fugue 0.9.1__py3-none-any.whl → 0.9.2.dev2__py3-none-any.whl

fugue/dataframe/function_wrapper.py

@@ -80,6 +80,7 @@ class DataFrameFunctionWrapper(FunctionWrapper):
         p.update(kwargs)
         has_kw = False
         rargs: Dict[str, Any] = {}
+        row_param_info: Any = None
         for k, v in self._params.items():
             if isinstance(v, (PositionalParam, KeywordParam)):
                 if isinstance(v, KeywordParam):
@@ -90,7 +91,14 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    rargs[k] = v.to_input_data(p[k], ctx=ctx)
+                    if v.is_per_row:
+                        assert_or_throw(
+                            row_param_info is None,
+                            lambda: ValueError("only one row parameter is allowed"),
+                        )
+                        row_param_info = (k, v, p[k])
+                    else:
+                        rargs[k] = v.to_input_data(p[k], ctx=ctx)
                 else:
                     rargs[k] = p[k]  # TODO: should we do auto type conversion?
                 del p[k]
@@ -100,12 +108,38 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             rargs.update(p)
         elif not ignore_unknown and len(p) > 0:
             raise ValueError(f"{p} are not acceptable parameters")
+        if row_param_info is None:
+            return self._run_func(rargs, output, output_schema, ctx, raw=False)
+        else:  # input contains row parameter
+
+            def _dfs() -> Iterable[Any]:
+                k, v, df = row_param_info
+                for row in v.to_input_rows(df, ctx):
+                    rargs[k] = None
+                    _rargs = rargs.copy()
+                    _rargs[k] = row
+                    yield self._run_func(_rargs, output, output_schema, ctx, raw=True)
+
+            if not output:
+                sum(1 for _ in _dfs())
+                return
+            else:
+                return self._rt.iterable_to_output_df(_dfs(), output_schema, ctx)
+
+    def _run_func(
+        self,
+        rargs: Dict[str, Any],
+        output: bool,
+        output_schema: Any,
+        ctx: Any,
+        raw: bool,
+    ) -> Any:
         rt = self._func(**rargs)
         if not output:
             if isinstance(self._rt, _DataFrameParamBase):
                 self._rt.count(rt)
             return
-        if isinstance(self._rt, _DataFrameParamBase):
+        if not raw and isinstance(self._rt, _DataFrameParamBase):
             return self._rt.to_output_df(rt, output_schema, ctx=ctx)
         return rt
 
@@ -120,6 +154,7 @@ fugue_annotated_param = DataFrameFunctionWrapper.annotated_param
         annotation == Callable
         or annotation == callable  # pylint: disable=comparison-with-callable
         or str(annotation).startswith("typing.Callable")
+        or str(annotation).startswith("collections.abc.Callable")
     ),
 )
 class _CallableParam(AnnotatedParam):
@@ -134,6 +169,9 @@ class _CallableParam(AnnotatedParam):
         or annotation == Optional[callable]
         or str(annotation).startswith("typing.Union[typing.Callable")  # 3.8-
         or str(annotation).startswith("typing.Optional[typing.Callable")  # 3.9+
+        or str(annotation).startswith(
+            "typing.Optional[collections.abc.Callable]"
+        )  # 3.9+
     ),
 )
 class _OptionalCallableParam(AnnotatedParam):
@@ -145,14 +183,30 @@ class _DataFrameParamBase(AnnotatedParam):
         super().__init__(param)
         assert_or_throw(self.required, lambda: TypeError(f"{self} must be required"))
 
+    @property
+    def is_per_row(self) -> bool:
+        return False
+
     def to_input_data(self, df: DataFrame, ctx: Any) -> Any:  # pragma: no cover
         raise NotImplementedError
 
+    def to_input_rows(
+        self,
+        df: DataFrame,
+        ctx: Any,
+    ) -> Iterable[Any]:
+        raise NotImplementedError  # pragma: no cover
+
     def to_output_df(
         self, df: Any, schema: Any, ctx: Any
     ) -> DataFrame:  # pragma: no cover
         raise NotImplementedError
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        raise NotImplementedError
+
     def count(self, df: Any) -> int:  # pragma: no cover
         raise NotImplementedError
 
@@ -182,6 +236,34 @@ class DataFrameParam(_DataFrameParamBase):
             return sum(1 for _ in df.as_array_iterable())
 
 
+@fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
+class RowParam(_DataFrameParamBase):
+    @property
+    def is_per_row(self) -> bool:
+        return True
+
+    def count(self, df: Any) -> int:
+        return 1
+
+
+@fugue_annotated_param(Dict[str, Any])
+class DictParam(RowParam):
+    def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
+        yield from df.as_dict_iterable()
+
+    def to_output_df(self, output: Dict[str, Any], schema: Any, ctx: Any) -> DataFrame:
+        return ArrayDataFrame([list(output.values())], schema)
+
+    def iterable_to_output_df(
+        self, dfs: Iterable[Dict[str, Any]], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        params: Dict[str, Any] = {}
+        if schema is not None:
+            params["schema"] = Schema(schema).pa_schema
+        adf = pa.Table.from_pylist(list(dfs), **params)
+        return ArrowDataFrame(adf)
+
+
 @fugue_annotated_param(AnyDataFrame)
 class _AnyDataFrameParam(DataFrameParam):
     def to_output_df(self, output: AnyDataFrame, schema: Any, ctx: Any) -> DataFrame:
@@ -207,6 +289,15 @@ class LocalDataFrameParam(DataFrameParam):
         )
         return output
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        def _dfs() -> Iterable[DataFrame]:
+            for df in dfs:
+                yield self.to_output_df(df, schema, ctx)
+
+        return LocalDataFrameIterableDataFrame(_dfs(), schema=schema)
+
     def count(self, df: LocalDataFrame) -> int:
         if df.is_bounded:
             return df.count()

fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspq]$"
+            func, "^[lspqr][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspnq]$"
+            func, "^[lspqr][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -503,7 +503,7 @@ class _FuncAsCoTransformer(CoTransformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = schema  # type: ignore
@@ -562,7 +562,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
 
         tr = _FuncAsOutputCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = None  # type: ignore

{fugue-0.9.1.dist-info → fugue-0.9.2.dev2.dist-info}/METADATA

@@ -1,13 +1,12 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.9.1
+Version: 0.9.2.dev2
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
 Author-email: hello@fugue.ai
 License: Apache-2.0
 Keywords: distributed spark dask ray sql dsl domain specific language
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -17,67 +16,68 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad >=0.9.7
-Requires-Dist: adagio >=0.2.4
+License-File: LICENSE
+Requires-Dist: triad>=0.9.7
+Requires-Dist: adagio>=0.2.6
 Provides-Extra: all
-Requires-Dist: qpd >=0.4.4 ; extra == 'all'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'all'
-Requires-Dist: sqlglot ; extra == 'all'
-Requires-Dist: jinja2 ; extra == 'all'
-Requires-Dist: pyspark >=3.1.1 ; extra == 'all'
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'all'
-Requires-Dist: dask-sql ; extra == 'all'
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'all'
-Requires-Dist: notebook ; extra == 'all'
-Requires-Dist: jupyterlab ; extra == 'all'
-Requires-Dist: ipython >=7.10.0 ; extra == 'all'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'all'
-Requires-Dist: pyarrow >=6.0.1 ; extra == 'all'
-Requires-Dist: pandas <2.2,>=2.0.2 ; extra == 'all'
-Requires-Dist: ibis-framework ; extra == 'all'
-Requires-Dist: polars ; extra == 'all'
+Requires-Dist: qpd>=0.4.4; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: sqlglot; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: pyspark>=3.1.1; extra == "all"
+Requires-Dist: dask[dataframe,distributed]>=2023.5.0; extra == "all"
+Requires-Dist: dask-sql; extra == "all"
+Requires-Dist: ray[data]>=2.5.0; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Requires-Dist: ipython>=7.10.0; extra == "all"
+Requires-Dist: duckdb>=0.5.0; extra == "all"
+Requires-Dist: pyarrow>=6.0.1; extra == "all"
+Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
+Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
+Requires-Dist: polars; extra == "all"
 Provides-Extra: cpp_sql_parser
-Requires-Dist: fugue-sql-antlr[cpp] >=0.2.0 ; extra == 'cpp_sql_parser'
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
 Provides-Extra: dask
-Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'dask'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'dask'
-Requires-Dist: pandas >=2.0.2 ; extra == 'dask'
-Requires-Dist: dask[dataframe,distributed] >=2024.4.0 ; (python_version >= "3.11.9") and extra == 'dask'
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "dask"
+Requires-Dist: pyarrow>=7.0.0; extra == "dask"
+Requires-Dist: pandas>=2.0.2; extra == "dask"
 Provides-Extra: duckdb
-Requires-Dist: qpd >=0.4.4 ; extra == 'duckdb'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'duckdb'
-Requires-Dist: sqlglot ; extra == 'duckdb'
-Requires-Dist: jinja2 ; extra == 'duckdb'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'duckdb'
-Requires-Dist: numpy ; extra == 'duckdb'
+Requires-Dist: qpd>=0.4.4; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: sqlglot; extra == "duckdb"
+Requires-Dist: jinja2; extra == "duckdb"
+Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
+Requires-Dist: numpy; extra == "duckdb"
 Provides-Extra: ibis
-Requires-Dist: qpd >=0.4.4 ; extra == 'ibis'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'ibis'
-Requires-Dist: sqlglot ; extra == 'ibis'
-Requires-Dist: jinja2 ; extra == 'ibis'
-Requires-Dist: ibis-framework ; extra == 'ibis'
-Requires-Dist: pandas <2.2 ; extra == 'ibis'
+Requires-Dist: qpd>=0.4.4; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: sqlglot; extra == "ibis"
+Requires-Dist: jinja2; extra == "ibis"
+Requires-Dist: ibis-framework[pandas]; extra == "ibis"
+Requires-Dist: pandas<2.2; extra == "ibis"
 Provides-Extra: notebook
-Requires-Dist: notebook ; extra == 'notebook'
-Requires-Dist: jupyterlab ; extra == 'notebook'
-Requires-Dist: ipython >=7.10.0 ; extra == 'notebook'
+Requires-Dist: notebook; extra == "notebook"
+Requires-Dist: jupyterlab; extra == "notebook"
+Requires-Dist: ipython>=7.10.0; extra == "notebook"
 Provides-Extra: polars
-Requires-Dist: polars ; extra == 'polars'
+Requires-Dist: polars; extra == "polars"
 Provides-Extra: ray
-Requires-Dist: ray[data] >=2.5.0 ; extra == 'ray'
-Requires-Dist: duckdb >=0.5.0 ; extra == 'ray'
-Requires-Dist: pyarrow >=7.0.0 ; extra == 'ray'
-Requires-Dist: pandas <2.2 ; extra == 'ray'
+Requires-Dist: ray[data]>=2.5.0; extra == "ray"
+Requires-Dist: duckdb>=0.5.0; extra == "ray"
+Requires-Dist: pyarrow>=7.0.0; extra == "ray"
+Requires-Dist: pandas<2.2; extra == "ray"
 Provides-Extra: spark
-Requires-Dist: pyspark >=3.1.1 ; extra == 'spark'
+Requires-Dist: pyspark>=3.1.1; extra == "spark"
 Provides-Extra: sql
-Requires-Dist: qpd >=0.4.4 ; extra == 'sql'
-Requires-Dist: fugue-sql-antlr >=0.2.0 ; extra == 'sql'
-Requires-Dist: sqlglot ; extra == 'sql'
-Requires-Dist: jinja2 ; extra == 'sql'
+Requires-Dist: qpd>=0.4.4; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: sqlglot; extra == "sql"
+Requires-Dist: jinja2; extra == "sql"
 
 # Fugue
 
@@ -355,4 +355,3 @@ View some of our latest conferences presentations and content. For a more comple
 *   [Large Scale Data Validation with Spark and Dask (PyCon US)](https://www.youtube.com/watch?v=2AdvBgjO_3Q)
 *   [FugueSQL - The Enhanced SQL Interface for Pandas, Spark, and Dask DataFrames (PyData Global)](https://www.youtube.com/watch?v=OBpnGYjNBBI)
 *   [Distributed Hybrid Parameter Tuning](https://www.youtube.com/watch?v=_GBjqskD8Qk)
-

{fugue-0.9.1.dist-info → fugue-0.9.2.dev2.dist-info}/RECORD

@@ -31,7 +31,7 @@ fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvr
 fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=hOZF3GmwpxqwqKi9-pEOAPZSW1ZFyB47hLxRrGyOiuM,14855
+fugue/dataframe/function_wrapper.py,sha256=1CjI4UXHffomylK0_u0CGL1dPv_sSXTN22S5grD10_w,17889
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
 fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
@@ -61,7 +61,7 @@ fugue/extensions/processor/convert.py,sha256=zG0lMtHGwY5TsqK4eplbMdlTg7J_PD3HbI0
 fugue/extensions/processor/processor.py,sha256=czhQlQgMpAXXoLVAX9Q0TFUMYEEhsgufTammxcKSmOY,1665
 fugue/extensions/transformer/__init__.py,sha256=VD6d-8xW1Yl8fUPj43cBWNR9pCOlYD9xWyGIHAlHwvI,456
 fugue/extensions/transformer/constants.py,sha256=76DfpoTOGQ8gp5XtCs_xznfbr_H015-prXpHWSqMNDU,59
-fugue/extensions/transformer/convert.py,sha256=5fhktR2s13ZOpUihpy-gy7Xn2BRN6UoA5uwOzJ6YNOU,23380
+fugue/extensions/transformer/convert.py,sha256=SU_KvzZp_nV8oCxZGx7qDsdCE0CJ--8UAp5m8z4d4HY,23386
 fugue/extensions/transformer/transformer.py,sha256=zhOUgyv5-DPxYd1CP_98WeEw-zUgwknRnPW_6di-q3g,9098
 fugue/rpc/__init__.py,sha256=3GzUl4QZQuCChjD7eaTJW8tnTwfke6ZY9r9g5nCeBZ8,167
 fugue/rpc/base.py,sha256=3Fq5SvwLZqw9NXru3r32WuJKBGFr9bl7nFgy6e9boGo,8470
@@ -89,10 +89,11 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
+fugue_dask/_dask_sql_wrapper.py,sha256=lj38gJIOdoMV9W44gpwzLjUEtPVsQNKjRWuEkfI7-PM,2618
 fugue_dask/_io.py,sha256=pl4F7mbVgP7Rwh1FFG7xfOz2TBZRUj1l3lLvDY4jOf4,6020
-fugue_dask/_utils.py,sha256=1uplEqvpCDZDp2YdwJxa6cuGScpgG9VvN3057J02bys,8956
+fugue_dask/_utils.py,sha256=dGUkhOoXQqgGQH_BY6aeYFo9UIWUAyo8YjwtdB7QD4s,8951
 fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=60IiwYRBVhN-pX3v6i9BZ8Pa4bcSh5UoklvCScM_XAM,21361
+fugue_dask/execution_engine.py,sha256=Em9pN6cw5w5DGLcjV6oKQKQeLLblc9DZ0DkvxKVFxQQ,21167
 fugue_dask/registry.py,sha256=jepWKH55VWNIWV3pOF5vpCl2OpO0rI1IULx5GM2Gk6w,2274
 fugue_dask/tester.py,sha256=E7BZjgFpJgrHsLMKzvSO5im5OwocYcratjzulJSQZl0,718
 fugue_duckdb/__init__.py,sha256=ZzhmAWbROR1YL9Kmlt7OlwkgPZzFhsSdwLV2pFmAqGI,268
@@ -107,7 +108,7 @@ fugue_ibis/__init__.py,sha256=z7TkK7M2_0p9XO6jQATNDgT0aHXn5k69Ttz2ga-eQG8,190
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
 fugue_ibis/dataframe.py,sha256=k4Q6qBLBIADF5YhbvaDplXO7OkMZSHuf_Wg5o-AusEI,7796
-fugue_ibis/execution_engine.py,sha256=5I-ou5xPdomVu-srdvidvP8f7wDYbGrCV_lGffZa_ac,18679
+fugue_ibis/execution_engine.py,sha256=jRnp1m1wuTicS29A-WA043f8QwdoK8b9rwPXvTkm8r8,18751
 fugue_notebook/__init__.py,sha256=9r_-2uxu1lBeZ8GgpYCKom_OZy2soIOYZajg7JDO-HY,4326
 fugue_notebook/env.py,sha256=TYiTxYPFi-BVJJY49jDsvw9mddhK8WrifeRxBke30I8,4773
 fugue_notebook/nbextension/README.md,sha256=QLnr957YeGfwzy2r4c4qbZPaXyCbyGrKPvcqSBQYSnU,123
@@ -127,7 +128,7 @@ fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
-fugue_ray/_utils/io.py,sha256=3hFNDeBuh4bfCud40ZsGrGZLSvCSuxL_1VlqCTnn6RA,9794
+fugue_ray/_utils/io.py,sha256=Dz0WuQrh_8Ix7jU5viFIA6caJcfxV4ew0ruBZLQbD1s,9930
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
@@ -143,14 +144,14 @@ fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=cOkZG6w1RHhWWxtjQhZClZQaGT6haNd576BoUmNC_cA,77960
+fugue_test/builtin_suite.py,sha256=BpGwa66cAUuuc7ULOsPP3ax8IKQtNIPoSmlUFgqUKQk,79252
 fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
 fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
 fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
-fugue_version/__init__.py,sha256=UwJXM8JY2T3tE2id0K2k_lEaVThbRTrGO1mNibyzIz8,22
-fugue-0.9.1.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.9.1.dist-info/METADATA,sha256=zu44QGPIwk28QyKe9H4Si2ANByy1sJ9cmauNrhCg4bc,18380
-fugue-0.9.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fugue-0.9.1.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
-fugue-0.9.1.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.9.1.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
+fugue-0.9.2.dev2.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.9.2.dev2.dist-info/METADATA,sha256=eR5mL6Tf1RGa_-Do5Dmzy4ZkbcbKf-FzW4qA0cAW1Ec,18283
+fugue-0.9.2.dev2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+fugue-0.9.2.dev2.dist-info/entry_points.txt,sha256=2Vxp1qew_tswacA8m0RzIliLlFOQMlzezvSXPugM_KA,295
+fugue-0.9.2.dev2.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.2.dev2.dist-info/RECORD,,

{fugue-0.9.1.dist-info → fugue-0.9.2.dev2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fugue-0.9.2.dev2.dist-info/entry_points.txt

@@ -0,0 +1,11 @@
+[fugue.plugins]
+dask = fugue_dask.registry[dask]
+duckdb = fugue_duckdb.registry[duckdb]
+ibis = fugue_ibis[ibis]
+polars = fugue_polars.registry[polars]
+ray = fugue_ray.registry[ray]
+spark = fugue_spark.registry[spark]
+
+[pytest11]
+fugue_test = fugue_test
+fugue_test_fixtures = fugue_test.fixtures

fugue_dask/_dask_sql_wrapper.py

@@ -0,0 +1,76 @@
+from typing import Any, Optional
+
+import dask.dataframe as dd
+
+try:
+    from dask.dataframe.dask_expr.io.parquet import ReadParquet
+
+    HAS_DASK_EXPR = True  # newer dask
+except ImportError:  # pragma: no cover
+    HAS_DASK_EXPR = False  # older dask
+
+if not HAS_DASK_EXPR:  # pragma: no cover
+    try:
+        from dask_sql import Context as ContextWrapper  # pylint: disable-all
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+else:
+    from triad.utils.assertion import assert_or_throw
+
+    try:
+        from dask_sql import Context
+        from dask_sql.datacontainer import Statistics
+        from dask_sql.input_utils import InputUtil
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "dask-sql is not installed. Please install it with `pip install dask-sql`"
+        )
+
+    class ContextWrapper(Context):  # type: ignore
+        def create_table(
+            self,
+            table_name: str,
+            input_table: dd.DataFrame,
+            format: Optional[str] = None,  # noqa
+            persist: bool = False,
+            schema_name: Optional[str] = None,
+            statistics: Optional[Statistics] = None,
+            gpu: bool = False,
+            **kwargs: Any,
+        ) -> None:  # pragma: no cover
+            assert_or_throw(
+                isinstance(input_table, dd.DataFrame),
+                lambda: ValueError(
+                    f"input_table must be a dask dataframe, but got {type(input_table)}"
+                ),
+            )
+            assert_or_throw(
+                dd._dask_expr_enabled(), lambda: ValueError("Dask expr must be enabled")
+            )
+            schema_name = schema_name or self.schema_name
+
+            dc = InputUtil.to_dc(
+                input_table,
+                table_name=table_name,
+                format=format,
+                persist=persist,
+                gpu=gpu,
+                **kwargs,
+            )
+
+            dask_filepath = None
+            operations = input_table.find_operations(ReadParquet)
+            for op in operations:
+                dask_filepath = op._args[0]
+
+            dc.filepath = dask_filepath
+            self.schema[schema_name].filepaths[table_name.lower()] = dask_filepath
+
+            if not statistics:
+                statistics = Statistics(float("nan"))
+            dc.statistics = statistics
+
+            self.schema[schema_name].tables[table_name.lower()] = dc
+            self.schema[schema_name].statistics[table_name.lower()] = statistics

fugue_dask/_utils.py

@@ -5,7 +5,7 @@ import dask.dataframe as dd
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-from dask.dataframe.core import DataFrame
+from dask.dataframe import DataFrame
 from dask.delayed import delayed
 from dask.distributed import Client, get_client
 from triad.utils.pandas_like import PD_UTILS, PandasLikeUtils

fugue_dask/execution_engine.py

@@ -9,9 +9,10 @@ from triad.collections import Schema
 from triad.collections.dict import IndexedOrderedDict, ParamDict
 from triad.utils.assertion import assert_or_throw
 from triad.utils.hash import to_uuid
+from triad.utils.io import makedirs
 from triad.utils.pandas_like import PandasUtils
 from triad.utils.threading import RunOnce
-from triad.utils.io import makedirs
+
 from fugue import StructuredRawSQL
 from fugue.collections.partition import (
     PartitionCursor,
@@ -61,14 +62,9 @@ class DaskSQLEngine(SQLEngine):
         return True
 
     def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        try:
-            from dask_sql import Context
-        except ImportError:  # pragma: no cover
-            raise ImportError(
-                "dask-sql is not installed. "
-                "Please install it with `pip install dask-sql`"
-            )
-        ctx = Context()
+        from ._dask_sql_wrapper import ContextWrapper
+
+        ctx = ContextWrapper()
         _dfs: Dict[str, dd.DataFrame] = {k: self._to_safe_df(v) for k, v in dfs.items()}
         sql = statement.construct(dialect=self.dialect, log=self.log)
         res = ctx.sql(

fugue_ibis/execution_engine.py

@@ -92,7 +92,8 @@ class IbisSQLEngine(SQLEngine):
         _df2 = self.to_df(df2)
         key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
         on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
-        if ibis.__version__ < "6":  # pragma: no cover
+        version = int(ibis.__version__.split(".")[0])
+        if version < 6:  # pragma: no cover
             suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
         else:
             # breaking change in ibis 6.0
@@ -113,7 +114,7 @@ class IbisSQLEngine(SQLEngine):
                     cols.append(
                         ibis.coalesce(tb[k], tb[k + _JOIN_RIGHT_SUFFIX]).name(k)
                     )
-            tb = tb[cols]
+            tb = tb.select(*cols)
         elif how.lower() in ["semi", "left_semi"]:
             tb = _df1.native.semi_join(_df2.native, on_fields, **suffixes)
         elif how.lower() in ["anti", "left_anti"]:
@@ -153,7 +154,7 @@ class IbisSQLEngine(SQLEngine):
         self,
         df: DataFrame,
         how: str = "any",
-        thresh: int = None,
+        thresh: Optional[int] = None,
         subset: Optional[List[str]] = None,
     ) -> DataFrame:
         schema = df.schema
@@ -161,7 +162,7 @@ class IbisSQLEngine(SQLEngine):
             schema = schema.extract(subset)
         _df = self.to_df(df)
         if thresh is None:
-            tb = _df.native.dropna(subset=subset, how=how)
+            tb = _df.native.drop_null(subset, how=how)
             return self.to_df(tb, df.schema)
         assert_or_throw(
             how == "any", ValueError("when thresh is set, how must be 'any'")
@@ -204,7 +205,7 @@ class IbisSQLEngine(SQLEngine):
             ibis.coalesce(tb[f], ibis.literal(vd[f])).name(f) if f in names else tb[f]
             for f in df.columns
         ]
-        return self.to_df(tb[cols], schema=df.schema)
+        return self.to_df(tb.select(cols), schema=df.schema)
 
     def take(
         self,
@@ -241,7 +242,7 @@ class IbisSQLEngine(SQLEngine):
                 f") WHERE __fugue_take_param<={n}"
             )
             tb = self.query_to_table(sql, {tbn: idf})
-            return self.to_df(tb[df.columns], schema=df.schema)
+            return self.to_df(tb.select(*df.columns), schema=df.schema)
 
         sorts: List[str] = []
         for k, v in _presort.items():

fugue_ray/_utils/io.py

@@ -7,7 +7,7 @@ import ray.data as rd
 from packaging import version
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
-from ray.data.datasource import FileExtensionFilter
+
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
@@ -21,6 +21,27 @@ from fugue_ray.dataframe import RayDataFrame
 
 from .._constants import RAY_VERSION
 
+try:
+    from ray.data.datasource import FileExtensionFilter
+
+    class _FileFiler(FileExtensionFilter):  # pragma: no cover
+        def __init__(
+            self, file_extensions: Union[str, List[str]], exclude: Iterable[str]
+        ):
+            super().__init__(file_extensions, allow_if_no_extension=True)
+            self._exclude = set(exclude)
+
+        def _is_valid(self, path: str) -> bool:
+            return pathlib.Path(
+                path
+            ).name not in self._exclude and self._file_has_extension(path)
+
+        def __call__(self, paths: List[str]) -> List[str]:
+            return [path for path in paths if self._is_valid(path)]
+
+except ImportError:  # pragma: no cover
+    pass  # ray >=2.10
+
 
 class RayIO(object):
     def __init__(self, engine: ExecutionEngine):
@@ -248,17 +269,3 @@ class RayIO(object):
 
     def _remote_args(self) -> Dict[str, Any]:
         return {"num_cpus": 1}
-
-
-class _FileFiler(FileExtensionFilter):  # pragma: no cover
-    def __init__(self, file_extensions: Union[str, List[str]], exclude: Iterable[str]):
-        super().__init__(file_extensions, allow_if_no_extension=True)
-        self._exclude = set(exclude)
-
-    def _is_valid(self, path: str) -> bool:
-        return pathlib.Path(
-            path
-        ).name not in self._exclude and self._file_has_extension(path)
-
-    def __call__(self, paths: List[str]) -> List[str]:
-        return [path for path in paths if self._is_valid(path)]

fugue_test/builtin_suite.py

@@ -486,6 +486,23 @@ class BuiltInTests(object):
                 dag.df([], "a:int,b:int").assert_eq(b)
             dag.run(self.engine)
 
+        def test_transform_row_wise(self):
+            def t1(row: Dict[str, Any]) -> Dict[str, Any]:
+                row["b"] = 1
+                return row
+
+            def t2(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
+                return rows[0]
+
+            with fa.engine_context(self.engine):
+                a = pd.DataFrame([[3, 4], [1, 2], [3, 5]], columns=["a", "b"])
+                b = fa.transform(a, t1, schema="*")
+                assert sorted(fa.as_array(b)) == [[1, 1], [3, 1], [3, 1]]
+                b = fa.transform(
+                    a, t2, schema="*", partition={"by": "a", "presort": "b"}
+                )
+                assert sorted(fa.as_array(b)) == [[1, 2], [3, 4]]
+
         def test_transform_binary(self):
             with FugueWorkflow() as dag:
                 a = dag.df([[1, pickle.dumps([0, "a"])]], "a:int,b:bytes")
@@ -548,6 +565,8 @@ class BuiltInTests(object):
                 e = dag.df([[1, 2, 1, 10]], "a:int,ct1:int,ct2:int,x:int")
                 e.assert_eq(c)
 
+                a.zip(b).transform(mock_co_tf1_d, params=dict(p=10)).assert_eq(e)
+
                 # interfaceless
                 c = dag.transform(
                     a.zip(b),
@@ -676,6 +695,13 @@ class BuiltInTests(object):
                 incr()
                 yield pa.Table.from_pandas(df)
 
+            def t11(row: Dict[str, Any]) -> Dict[str, Any]:
+                incr()
+                return row
+
+            def t12(row: Dict[str, Any]) -> None:
+                incr()
+
             with FugueWorkflow() as dag:
                 a = dag.df([[1, 2], [3, 4]], "a:double,b:int")
                 a.out_transform(t1)  # +2
@@ -688,6 +714,8 @@ class BuiltInTests(object):
                 a.out_transform(t8, ignore_errors=[NotImplementedError])  # +1
                 a.out_transform(t9)  # +1
                 a.out_transform(t10)  # +1
+                a.out_transform(t11)  # +2
+                a.out_transform(t12)  # +2
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t2))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t3))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t4))
@@ -695,7 +723,7 @@ class BuiltInTests(object):
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(T7))
             dag.run(self.engine)
 
-            assert 13 <= incr()
+            assert 17 <= incr()
 
         def test_out_cotransform(self):  # noqa: C901
             tmpdir = str(self.tmpdir)
@@ -2001,6 +2029,13 @@ def mock_co_tf1(
     return [[df1[0]["a"], len(df1), len(df2), p]]
 
 
+@cotransformer(lambda dfs, **kwargs: "a:int,ct1:int,ct2:int,x:int")
+def mock_co_tf1_d(
+    df1: List[Dict[str, Any]], df2: List[List[Any]], p=1
+) -> Dict[str, Any]:
+    return dict(a=df1[0]["a"], ct1=len(df1), ct2=len(df2), x=p)
+
+
 def mock_co_tf2(dfs: DataFrames, p=1) -> List[List[Any]]:
     return [[dfs[0].peek_dict()["a"], dfs[0].count(), dfs[1].count(), p]]
 

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.1"
+__version__ = "0.9.2"

fugue-0.9.1.dist-info/entry_points.txt

@@ -1,12 +0,0 @@
-[fugue.plugins]
-dask = fugue_dask.registry [dask]
-duckdb = fugue_duckdb.registry [duckdb]
-ibis = fugue_ibis [ibis]
-polars = fugue_polars.registry [polars]
-ray = fugue_ray.registry [ray]
-spark = fugue_spark.registry [spark]
-
-[pytest11]
-fugue_test = fugue_test
-fugue_test_fixtures = fugue_test.fixtures
-