fugue 0.9.0.dev4__py3-none-any.whl → 0.9.2.dev1__py3-none-any.whl

fugue/dataframe/function_wrapper.py

@@ -20,6 +20,7 @@ from triad.collections.function_wrapper import (
     PositionalParam,
     function_wrapper,
 )
+from triad.utils.convert import compare_annotations
 from triad.utils.iter import EmptyAwareIterable, make_empty_aware
 
 from ..constants import FUGUE_ENTRYPOINT
@@ -37,6 +38,14 @@ from .iterable_dataframe import IterableDataFrame
 from .pandas_dataframe import PandasDataFrame
 
 
+def _compare_iter(tp: Any) -> Any:
+    return lambda x: compare_annotations(
+        x, Iterable[tp]  # type:ignore
+    ) or compare_annotations(
+        x, Iterator[tp]  # type:ignore
+    )
+
+
 @function_wrapper(FUGUE_ENTRYPOINT)
 class DataFrameFunctionWrapper(FunctionWrapper):
     @property
@@ -71,6 +80,7 @@ class DataFrameFunctionWrapper(FunctionWrapper):
         p.update(kwargs)
         has_kw = False
         rargs: Dict[str, Any] = {}
+        row_param_info: Any = None
         for k, v in self._params.items():
             if isinstance(v, (PositionalParam, KeywordParam)):
                 if isinstance(v, KeywordParam):
@@ -81,7 +91,14 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    rargs[k] = v.to_input_data(p[k], ctx=ctx)
+                    if v.is_per_row:
+                        assert_or_throw(
+                            row_param_info is None,
+                            lambda: ValueError("only one row parameter is allowed"),
+                        )
+                        row_param_info = (k, v, p[k])
+                    else:
+                        rargs[k] = v.to_input_data(p[k], ctx=ctx)
                 else:
                     rargs[k] = p[k]  # TODO: should we do auto type conversion?
                 del p[k]
@@ -91,12 +108,38 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             rargs.update(p)
         elif not ignore_unknown and len(p) > 0:
             raise ValueError(f"{p} are not acceptable parameters")
+        if row_param_info is None:
+            return self._run_func(rargs, output, output_schema, ctx, raw=False)
+        else:  # input contains row parameter
+
+            def _dfs() -> Iterable[Any]:
+                k, v, df = row_param_info
+                for row in v.to_input_rows(df, ctx):
+                    rargs[k] = None
+                    _rargs = rargs.copy()
+                    _rargs[k] = row
+                    yield self._run_func(_rargs, output, output_schema, ctx, raw=True)
+
+            if not output:
+                sum(1 for _ in _dfs())
+                return
+            else:
+                return self._rt.iterable_to_output_df(_dfs(), output_schema, ctx)
+
+    def _run_func(
+        self,
+        rargs: Dict[str, Any],
+        output: bool,
+        output_schema: Any,
+        ctx: Any,
+        raw: bool,
+    ) -> Any:
         rt = self._func(**rargs)
         if not output:
             if isinstance(self._rt, _DataFrameParamBase):
                 self._rt.count(rt)
             return
-        if isinstance(self._rt, _DataFrameParamBase):
+        if not raw and isinstance(self._rt, _DataFrameParamBase):
             return self._rt.to_output_df(rt, output_schema, ctx=ctx)
         return rt
 
@@ -136,14 +179,30 @@ class _DataFrameParamBase(AnnotatedParam):
         super().__init__(param)
         assert_or_throw(self.required, lambda: TypeError(f"{self} must be required"))
 
+    @property
+    def is_per_row(self) -> bool:
+        return False
+
     def to_input_data(self, df: DataFrame, ctx: Any) -> Any:  # pragma: no cover
         raise NotImplementedError
 
+    def to_input_rows(
+        self,
+        df: DataFrame,
+        ctx: Any,
+    ) -> Iterable[Any]:
+        raise NotImplementedError  # pragma: no cover
+
     def to_output_df(
         self, df: Any, schema: Any, ctx: Any
     ) -> DataFrame:  # pragma: no cover
         raise NotImplementedError
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        raise NotImplementedError
+
     def count(self, df: Any) -> int:  # pragma: no cover
         raise NotImplementedError
 
@@ -173,6 +232,34 @@ class DataFrameParam(_DataFrameParamBase):
             return sum(1 for _ in df.as_array_iterable())
 
 
+@fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
+class RowParam(_DataFrameParamBase):
+    @property
+    def is_per_row(self) -> bool:
+        return True
+
+    def count(self, df: Any) -> int:
+        return 1
+
+
+@fugue_annotated_param(Dict[str, Any])
+class DictParam(RowParam):
+    def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
+        yield from df.as_dict_iterable()
+
+    def to_output_df(self, output: Dict[str, Any], schema: Any, ctx: Any) -> DataFrame:
+        return ArrayDataFrame([list(output.values())], schema)
+
+    def iterable_to_output_df(
+        self, dfs: Iterable[Dict[str, Any]], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        params: Dict[str, Any] = {}
+        if schema is not None:
+            params["schema"] = Schema(schema).pa_schema
+        adf = pa.Table.from_pylist(list(dfs), **params)
+        return ArrowDataFrame(adf)
+
+
 @fugue_annotated_param(AnyDataFrame)
 class _AnyDataFrameParam(DataFrameParam):
     def to_output_df(self, output: AnyDataFrame, schema: Any, ctx: Any) -> DataFrame:
@@ -198,6 +285,15 @@ class LocalDataFrameParam(DataFrameParam):
         )
         return output
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        def _dfs() -> Iterable[DataFrame]:
+            for df in dfs:
+                yield self.to_output_df(df, schema, ctx)
+
+        return LocalDataFrameIterableDataFrame(_dfs(), schema=schema)
+
     def count(self, df: LocalDataFrame) -> int:
         if df.is_bounded:
             return df.count()
@@ -228,10 +324,7 @@ class _ListListParam(_LocalNoSchemaDataFrameParam):
         return len(df)
 
 
-@fugue_annotated_param(
-    Iterable[List[Any]],
-    matcher=lambda x: x == Iterable[List[Any]] or x == Iterator[List[Any]],
-)
+@fugue_annotated_param(Iterable[List[Any]], matcher=_compare_iter(List[Any]))
 class _IterableListParam(_LocalNoSchemaDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[List[Any]]:
@@ -288,10 +381,7 @@ class _ListDictParam(_LocalNoSchemaDataFrameParam):
         return len(df)
 
 
-@fugue_annotated_param(
-    Iterable[Dict[str, Any]],
-    matcher=lambda x: x == Iterable[Dict[str, Any]] or x == Iterator[Dict[str, Any]],
-)
+@fugue_annotated_param(Iterable[Dict[str, Any]], matcher=_compare_iter(Dict[str, Any]))
 class _IterableDictParam(_LocalNoSchemaDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[Dict[str, Any]]:
@@ -360,10 +450,7 @@ class _PandasParam(LocalDataFrameParam):
         return "pandas"
 
 
-@fugue_annotated_param(
-    Iterable[pd.DataFrame],
-    matcher=lambda x: x == Iterable[pd.DataFrame] or x == Iterator[pd.DataFrame],
-)
+@fugue_annotated_param(Iterable[pd.DataFrame], matcher=_compare_iter(pd.DataFrame))
 class _IterablePandasParam(LocalDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[pd.DataFrame]:
@@ -419,10 +506,7 @@ class _PyArrowTableParam(LocalDataFrameParam):
         return "pyarrow"
 
 
-@fugue_annotated_param(
-    Iterable[pa.Table],
-    matcher=lambda x: x == Iterable[pa.Table] or x == Iterator[pa.Table],
-)
+@fugue_annotated_param(Iterable[pa.Table], matcher=_compare_iter(pa.Table))
 class _IterableArrowParam(LocalDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[pa.Table]:

fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspq]$"
+            func, "^[lspqr][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspnq]$"
+            func, "^[lspqr][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -503,7 +503,7 @@ class _FuncAsCoTransformer(CoTransformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = schema  # type: ignore
@@ -562,7 +562,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
 
         tr = _FuncAsOutputCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = None  # type: ignore

{fugue-0.9.0.dev4.dist-info → fugue-0.9.2.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.9.0.dev4
+Version: 0.9.2.dev1
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad >=0.9.6
+Requires-Dist: triad >=0.9.7
 Requires-Dist: adagio >=0.2.4
 Provides-Extra: all
 Requires-Dist: qpd >=0.4.4 ; extra == 'all'

{fugue-0.9.0.dev4.dist-info → fugue-0.9.2.dev1.dist-info}/RECORD

@@ -31,7 +31,7 @@ fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvr
 fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=V1eQMOn27UroEYT7_YiwoEF0RjZYIM0zkD3vfaMAQFs,14813
+fugue/dataframe/function_wrapper.py,sha256=cG-0ICf-WxgxmEJIdU2jx4GzYhVfP69AyzjTjNDpXGE,17710
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
 fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
@@ -61,7 +61,7 @@ fugue/extensions/processor/convert.py,sha256=zG0lMtHGwY5TsqK4eplbMdlTg7J_PD3HbI0
 fugue/extensions/processor/processor.py,sha256=czhQlQgMpAXXoLVAX9Q0TFUMYEEhsgufTammxcKSmOY,1665
 fugue/extensions/transformer/__init__.py,sha256=VD6d-8xW1Yl8fUPj43cBWNR9pCOlYD9xWyGIHAlHwvI,456
 fugue/extensions/transformer/constants.py,sha256=76DfpoTOGQ8gp5XtCs_xznfbr_H015-prXpHWSqMNDU,59
-fugue/extensions/transformer/convert.py,sha256=5fhktR2s13ZOpUihpy-gy7Xn2BRN6UoA5uwOzJ6YNOU,23380
+fugue/extensions/transformer/convert.py,sha256=SU_KvzZp_nV8oCxZGx7qDsdCE0CJ--8UAp5m8z4d4HY,23386
 fugue/extensions/transformer/transformer.py,sha256=zhOUgyv5-DPxYd1CP_98WeEw-zUgwknRnPW_6di-q3g,9098
 fugue/rpc/__init__.py,sha256=3GzUl4QZQuCChjD7eaTJW8tnTwfke6ZY9r9g5nCeBZ8,167
 fugue/rpc/base.py,sha256=3Fq5SvwLZqw9NXru3r32WuJKBGFr9bl7nFgy6e9boGo,8470
@@ -127,7 +127,7 @@ fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
-fugue_ray/_utils/io.py,sha256=3hFNDeBuh4bfCud40ZsGrGZLSvCSuxL_1VlqCTnn6RA,9794
+fugue_ray/_utils/io.py,sha256=Dz0WuQrh_8Ix7jU5viFIA6caJcfxV4ew0ruBZLQbD1s,9930
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
@@ -137,20 +137,20 @@ fugue_spark/tester.py,sha256=VX003yGNlBukaZTQSN-w7XvgSk4rqxrWQIzno0dWrXg,2481
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
 fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
-fugue_spark/_utils/misc.py,sha256=o8dZmXOHnA7D_ps37vgGXTPTiSEG9LQzPKq7l-MG-qM,860
+fugue_spark/_utils/misc.py,sha256=9LsbBp6nOEhqXFLr8oWTc3VKzKk-vuVyixlRoquGnEs,858
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
 fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=cOkZG6w1RHhWWxtjQhZClZQaGT6haNd576BoUmNC_cA,77960
+fugue_test/builtin_suite.py,sha256=BpGwa66cAUuuc7ULOsPP3ax8IKQtNIPoSmlUFgqUKQk,79252
 fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
 fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
 fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
-fugue_version/__init__.py,sha256=H9NWRZb7NbeRRPLP_V1fARmLNXranorVM-OOY-8_2ug,22
-fugue-0.9.0.dev4.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.9.0.dev4.dist-info/METADATA,sha256=smbI6QuuMajmoMhJ14Y4MUs2mGpb4onc6kImR83D9DQ,18385
-fugue-0.9.0.dev4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fugue-0.9.0.dev4.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
-fugue-0.9.0.dev4.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.9.0.dev4.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
+fugue-0.9.2.dev1.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.9.2.dev1.dist-info/METADATA,sha256=0ML_xHPma0CaGuGtZn45doFbLu2UtzB-VlPX4koFLYg,18385
+fugue-0.9.2.dev1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+fugue-0.9.2.dev1.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
+fugue-0.9.2.dev1.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.2.dev1.dist-info/RECORD,,

fugue_ray/_utils/io.py

@@ -7,7 +7,7 @@ import ray.data as rd
 from packaging import version
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
-from ray.data.datasource import FileExtensionFilter
+
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
@@ -21,6 +21,27 @@ from fugue_ray.dataframe import RayDataFrame
 
 from .._constants import RAY_VERSION
 
+try:
+    from ray.data.datasource import FileExtensionFilter
+
+    class _FileFiler(FileExtensionFilter):  # pragma: no cover
+        def __init__(
+            self, file_extensions: Union[str, List[str]], exclude: Iterable[str]
+        ):
+            super().__init__(file_extensions, allow_if_no_extension=True)
+            self._exclude = set(exclude)
+
+        def _is_valid(self, path: str) -> bool:
+            return pathlib.Path(
+                path
+            ).name not in self._exclude and self._file_has_extension(path)
+
+        def __call__(self, paths: List[str]) -> List[str]:
+            return [path for path in paths if self._is_valid(path)]
+
+except ImportError:  # pragma: no cover
+    pass  # ray >=2.10
+
 
 class RayIO(object):
     def __init__(self, engine: ExecutionEngine):
@@ -248,17 +269,3 @@ class RayIO(object):
 
     def _remote_args(self) -> Dict[str, Any]:
         return {"num_cpus": 1}
-
-
-class _FileFiler(FileExtensionFilter):  # pragma: no cover
-    def __init__(self, file_extensions: Union[str, List[str]], exclude: Iterable[str]):
-        super().__init__(file_extensions, allow_if_no_extension=True)
-        self._exclude = set(exclude)
-
-    def _is_valid(self, path: str) -> bool:
-        return pathlib.Path(
-            path
-        ).name not in self._exclude and self._file_has_extension(path)
-
-    def __call__(self, paths: List[str]) -> List[str]:
-        return [path for path in paths if self._is_valid(path)]

fugue_spark/_utils/misc.py

@@ -3,7 +3,7 @@ from typing import Any
 try:
     from pyspark.sql.connect.session import SparkSession as SparkConnectSession
     from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
-except ImportError:  # pragma: no cover
+except Exception:  # pragma: no cover
     SparkConnectSession = None
     SparkConnectDataFrame = None
 import pyspark.sql as ps

fugue_test/builtin_suite.py

@@ -486,6 +486,23 @@ class BuiltInTests(object):
                 dag.df([], "a:int,b:int").assert_eq(b)
             dag.run(self.engine)
 
+        def test_transform_row_wise(self):
+            def t1(row: Dict[str, Any]) -> Dict[str, Any]:
+                row["b"] = 1
+                return row
+
+            def t2(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
+                return rows[0]
+
+            with fa.engine_context(self.engine):
+                a = pd.DataFrame([[3, 4], [1, 2], [3, 5]], columns=["a", "b"])
+                b = fa.transform(a, t1, schema="*")
+                assert sorted(fa.as_array(b)) == [[1, 1], [3, 1], [3, 1]]
+                b = fa.transform(
+                    a, t2, schema="*", partition={"by": "a", "presort": "b"}
+                )
+                assert sorted(fa.as_array(b)) == [[1, 2], [3, 4]]
+
         def test_transform_binary(self):
             with FugueWorkflow() as dag:
                 a = dag.df([[1, pickle.dumps([0, "a"])]], "a:int,b:bytes")
@@ -548,6 +565,8 @@ class BuiltInTests(object):
                 e = dag.df([[1, 2, 1, 10]], "a:int,ct1:int,ct2:int,x:int")
                 e.assert_eq(c)
 
+                a.zip(b).transform(mock_co_tf1_d, params=dict(p=10)).assert_eq(e)
+
                 # interfaceless
                 c = dag.transform(
                     a.zip(b),
@@ -676,6 +695,13 @@ class BuiltInTests(object):
                 incr()
                 yield pa.Table.from_pandas(df)
 
+            def t11(row: Dict[str, Any]) -> Dict[str, Any]:
+                incr()
+                return row
+
+            def t12(row: Dict[str, Any]) -> None:
+                incr()
+
             with FugueWorkflow() as dag:
                 a = dag.df([[1, 2], [3, 4]], "a:double,b:int")
                 a.out_transform(t1)  # +2
@@ -688,6 +714,8 @@ class BuiltInTests(object):
                 a.out_transform(t8, ignore_errors=[NotImplementedError])  # +1
                 a.out_transform(t9)  # +1
                 a.out_transform(t10)  # +1
+                a.out_transform(t11)  # +2
+                a.out_transform(t12)  # +2
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t2))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t3))
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(t4))
@@ -695,7 +723,7 @@ class BuiltInTests(object):
                 raises(FugueWorkflowCompileValidationError, lambda: a.out_transform(T7))
             dag.run(self.engine)
 
-            assert 13 <= incr()
+            assert 17 <= incr()
 
         def test_out_cotransform(self):  # noqa: C901
             tmpdir = str(self.tmpdir)
@@ -2001,6 +2029,13 @@ def mock_co_tf1(
     return [[df1[0]["a"], len(df1), len(df2), p]]
 
 
+@cotransformer(lambda dfs, **kwargs: "a:int,ct1:int,ct2:int,x:int")
+def mock_co_tf1_d(
+    df1: List[Dict[str, Any]], df2: List[List[Any]], p=1
+) -> Dict[str, Any]:
+    return dict(a=df1[0]["a"], ct1=len(df1), ct2=len(df2), x=p)
+
+
 def mock_co_tf2(dfs: DataFrames, p=1) -> List[List[Any]]:
     return [[dfs[0].peek_dict()["a"], dfs[0].count(), dfs[1].count(), p]]
 

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.0"
+__version__ = "0.9.2"