fugue 0.9.0.dev4__tar.gz → 0.9.2__tar.gz

fugue-0.9.2/PKG-INFO

@@ -0,0 +1,370 @@
+Metadata-Version: 2.4
+Name: fugue
+Version: 0.9.2
+Summary: An abstraction layer for distributed computation
+Home-page: http://github.com/fugue-project/fugue
+Author: The Fugue Development Team
+Author-email: hello@fugue.ai
+License: Apache-2.0
+Keywords: distributed spark dask ray sql dsl domain specific language
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: triad>=1.0.0
+Requires-Dist: adagio>=0.2.6
+Provides-Extra: sql
+Requires-Dist: qpd>=0.4.4; extra == "sql"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "sql"
+Requires-Dist: sqlglot; extra == "sql"
+Requires-Dist: jinja2; extra == "sql"
+Provides-Extra: cpp-sql-parser
+Requires-Dist: fugue-sql-antlr[cpp]>=0.2.0; extra == "cpp-sql-parser"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.1.1; extra == "spark"
+Provides-Extra: dask
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "dask"
+Requires-Dist: pyarrow>=7.0.0; extra == "dask"
+Requires-Dist: pandas>=2.0.2; extra == "dask"
+Provides-Extra: ray
+Requires-Dist: ray[data]>=2.30.0; extra == "ray"
+Requires-Dist: duckdb>=0.5.0; extra == "ray"
+Requires-Dist: pyarrow>=7.0.0; extra == "ray"
+Requires-Dist: pandas<2.2; extra == "ray"
+Provides-Extra: duckdb
+Requires-Dist: qpd>=0.4.4; extra == "duckdb"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "duckdb"
+Requires-Dist: sqlglot; extra == "duckdb"
+Requires-Dist: jinja2; extra == "duckdb"
+Requires-Dist: duckdb>=0.5.0; extra == "duckdb"
+Requires-Dist: numpy; extra == "duckdb"
+Provides-Extra: polars
+Requires-Dist: polars; extra == "polars"
+Provides-Extra: ibis
+Requires-Dist: qpd>=0.4.4; extra == "ibis"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "ibis"
+Requires-Dist: sqlglot; extra == "ibis"
+Requires-Dist: jinja2; extra == "ibis"
+Requires-Dist: ibis-framework[pandas]; extra == "ibis"
+Requires-Dist: pandas<2.2; extra == "ibis"
+Provides-Extra: notebook
+Requires-Dist: notebook; extra == "notebook"
+Requires-Dist: jupyterlab; extra == "notebook"
+Requires-Dist: ipython>=7.10.0; extra == "notebook"
+Provides-Extra: all
+Requires-Dist: qpd>=0.4.4; extra == "all"
+Requires-Dist: fugue-sql-antlr>=0.2.0; extra == "all"
+Requires-Dist: sqlglot; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: pyspark>=3.1.1; extra == "all"
+Requires-Dist: dask[dataframe,distributed]>=2024.4.0; extra == "all"
+Requires-Dist: dask-sql; extra == "all"
+Requires-Dist: ray[data]>=2.30.0; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Requires-Dist: ipython>=7.10.0; extra == "all"
+Requires-Dist: duckdb>=0.5.0; extra == "all"
+Requires-Dist: pyarrow>=6.0.1; extra == "all"
+Requires-Dist: pandas<2.2,>=2.0.2; extra == "all"
+Requires-Dist: ibis-framework[duckdb,pandas]; extra == "all"
+Requires-Dist: polars; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Fugue
+
+[![PyPI version](https://badge.fury.io/py/fugue.svg)](https://pypi.python.org/pypi/fugue/)
+[![PyPI pyversions](https://img.shields.io/pypi/pyversions/fugue.svg)](https://pypi.python.org/pypi/fugue/)
+[![PyPI license](https://img.shields.io/pypi/l/fugue.svg)](https://pypi.python.org/pypi/fugue/)
+[![codecov](https://codecov.io/gh/fugue-project/fugue/branch/master/graph/badge.svg?token=ZO9YD5N3IA)](https://codecov.io/gh/fugue-project/fugue)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
+[![Downloads](https://static.pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
+
+| Tutorials                                                                                           | API Documentation                                                                     | Chat with us on slack!                                                                                                   |
+| --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
+| [![Jupyter Book Badge](https://jupyterbook.org/badge.svg)](https://fugue-tutorials.readthedocs.io/) | [![Doc](https://readthedocs.org/projects/fugue/badge)](https://fugue.readthedocs.org) | [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](http://slack.fugue.ai) |
+
+
+**Fugue is a unified interface for distributed computing that lets users execute Python, Pandas, and SQL code on Spark, Dask, and Ray with minimal rewrites**.
+
+Fugue is most commonly used for:
+
+*   **Parallelizing or scaling existing Python and Pandas code** by bringing it to Spark, Dask, or Ray with minimal rewrites.
+*   Using [FugueSQL](https://fugue-tutorials.readthedocs.io/tutorials/quick_look/ten_minutes_sql.html) to **define end-to-end workflows** on top of Pandas, Spark, and Dask DataFrames. FugueSQL is an enhanced SQL interface that can invoke Python code.
+
+To see how Fugue compares to other frameworks like dbt, Arrow, Ibis, PySpark Pandas, see the [comparisons](https://fugue-tutorials.readthedocs.io/#how-does-fugue-compare-to)
+
+## [Fugue API](https://fugue-tutorials.readthedocs.io/tutorials/quick_look/ten_minutes.html)
+
+The Fugue API is a collection of functions that are capable of running on Pandas, Spark, Dask, and Ray. The simplest way to use Fugue is the [`transform()` function](https://fugue-tutorials.readthedocs.io/tutorials/beginner/transform.html). This lets users parallelize the execution of a single function by bringing it to Spark, Dask, or Ray. In the example below, the `map_letter_to_food()` function takes in a mapping and applies it on a column. This is just Pandas and Python so far (without Fugue).
+
+```python
+import pandas as pd
+from typing import Dict
+
+input_df = pd.DataFrame({"id":[0,1,2], "value": (["A", "B", "C"])})
+map_dict = {"A": "Apple", "B": "Banana", "C": "Carrot"}
+
+def map_letter_to_food(df: pd.DataFrame, mapping: Dict[str, str]) -> pd.DataFrame:
+    df["value"] = df["value"].map(mapping)
+    return df
+```
+
+Now, the `map_letter_to_food()` function is brought to the Spark execution engine by invoking the `transform()` function of Fugue. The output `schema` and `params` are passed to the `transform()` call. The `schema` is needed because it's a requirement for distributed frameworks. A schema of `"*"` below means all input columns are in the output.
+
+```python
+from pyspark.sql import SparkSession
+from fugue import transform
+
+spark = SparkSession.builder.getOrCreate()
+sdf = spark.createDataFrame(input_df)
+
+out = transform(sdf,
+               map_letter_to_food,
+               schema="*",
+               params=dict(mapping=map_dict),
+               )
+# out is a Spark DataFrame
+out.show()
+```
+```rst
++---+------+
+| id| value|
++---+------+
+|  0| Apple|
+|  1|Banana|
+|  2|Carrot|
++---+------+
+```
+
+<details>
+  <summary>PySpark equivalent of Fugue transform()</summary>
+
+  ```python
+from typing import Iterator, Union
+from pyspark.sql.types import StructType
+from pyspark.sql import DataFrame, SparkSession
+
+spark_session = SparkSession.builder.getOrCreate()
+
+def mapping_wrapper(dfs: Iterator[pd.DataFrame], mapping):
+    for df in dfs:
+        yield map_letter_to_food(df, mapping)
+
+def run_map_letter_to_food(input_df: Union[DataFrame, pd.DataFrame], mapping):
+    # conversion
+    if isinstance(input_df, pd.DataFrame):
+        sdf = spark_session.createDataFrame(input_df.copy())
+    else:
+        sdf = input_df.copy()
+
+    schema = StructType(list(sdf.schema.fields))
+    return sdf.mapInPandas(lambda dfs: mapping_wrapper(dfs, mapping),
+                            schema=schema)
+
+result = run_map_letter_to_food(input_df, map_dict)
+result.show()
+  ```
+</details>
+
+This syntax is simpler, cleaner, and more maintainable than the PySpark equivalent. At the same time, no edits were made to the original Pandas-based function to bring it to Spark. It is still usable on Pandas DataFrames. Fugue `transform()` also supports Dask and Ray as execution engines alongside the default Pandas-based engine.
+
+The Fugue API has a broader collection of functions that are also compatible with Spark, Dask, and Ray. For example, we can use `load()` and `save()` to create an end-to-end workflow compatible with Spark, Dask, and Ray. For the full list of functions, see the [Top Level API](https://fugue.readthedocs.io/en/latest/top_api.html)
+
+```python
+import fugue.api as fa
+
+def run(engine=None):
+    with fa.engine_context(engine):
+        df = fa.load("/path/to/file.parquet")
+        out = fa.transform(df, map_letter_to_food, schema="*")
+        fa.save(out, "/path/to/output_file.parquet")
+
+run()                 # runs on Pandas
+run(engine="spark")   # runs on Spark
+run(engine="dask")    # runs on Dask
+```
+
+All functions underneath the context will run on the specified backend. This makes it easy to toggle between local execution, and distributed execution.
+
+## [FugueSQL](https://fugue-tutorials.readthedocs.io/tutorials/fugue_sql/index.html)
+
+FugueSQL is a SQL-based language capable of expressing end-to-end data workflows on top of Pandas, Spark, and Dask. The `map_letter_to_food()` function above is used in the SQL expression below. This is how to use a Python-defined function along with the standard SQL `SELECT` statement.
+
+```python
+from fugue.api import fugue_sql
+import json
+
+query = """
+    SELECT id, value
+      FROM input_df
+    TRANSFORM USING map_letter_to_food(mapping={{mapping}}) SCHEMA *
+    """
+map_dict_str = json.dumps(map_dict)
+
+# returns Pandas DataFrame
+fugue_sql(query,mapping=map_dict_str)
+
+# returns Spark DataFrame
+fugue_sql(query, mapping=map_dict_str, engine="spark")
+```
+
+## Installation
+
+Fugue can be installed through pip or conda. For example:
+
+```bash
+pip install fugue
+```
+
+In order to use Fugue SQL, it is strongly recommended to install the `sql` extra:
+
+```bash
+pip install fugue[sql]
+```
+
+It also has the following installation extras:
+
+*   **sql**: to support Fugue SQL. Without this extra, the non-SQL part still works. Before Fugue 0.9.0, this extra is included in Fugue's core dependency so you don't need to install explicitly. **But for 0,9.0+, this becomes required if you want to use Fugue SQL.**
+*   **spark**: to support Spark as the [ExecutionEngine](https://fugue-tutorials.readthedocs.io/tutorials/advanced/execution_engine.html).
+*   **dask**: to support Dask as the ExecutionEngine.
+*   **ray**: to support Ray as the ExecutionEngine.
+*   **duckdb**: to support DuckDB as the ExecutionEngine, read [details](https://fugue-tutorials.readthedocs.io/tutorials/integrations/backends/duckdb.html).
+*   **polars**: to support Polars DataFrames and extensions using Polars.
+*   **ibis**: to enable Ibis for Fugue workflows, read [details](https://fugue-tutorials.readthedocs.io/tutorials/integrations/backends/ibis.html).
+*   **cpp_sql_parser**: to enable the CPP antlr parser for Fugue SQL. It can be 50+ times faster than the pure Python parser. For the main Python versions and platforms, there is already pre-built binaries, but for the remaining, it needs a C++ compiler to build on the fly.
+
+For example a common use case is:
+
+```bash
+pip install "fugue[duckdb,spark]"
+```
+
+Note if you already installed Spark or DuckDB independently, Fugue is able to automatically use them without installing the extras.
+
+## [Getting Started](https://fugue-tutorials.readthedocs.io/)
+
+The best way to get started with Fugue is to work through the 10 minute tutorials:
+
+*   [Fugue API in 10 minutes](https://fugue-tutorials.readthedocs.io/tutorials/quick_look/ten_minutes.html)
+*   [FugueSQL in 10 minutes](https://fugue-tutorials.readthedocs.io/tutorials/quick_look/ten_minutes_sql.html)
+
+For the top level API, see:
+
+*   [Fugue Top Level API](https://fugue.readthedocs.io/en/latest/top_api.html)
+
+The [tutorials](https://fugue-tutorials.readthedocs.io/) can also be run in an interactive notebook environment through binder or Docker:
+
+### Using binder
+
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fugue-project/tutorials/master)
+
+**Note it runs slow on binder** because the machine on binder isn't powerful enough for a distributed framework such as Spark. Parallel executions can become sequential, so some of the performance comparison examples will not give you the correct numbers.
+
+### Using Docker
+
+Alternatively, you should get decent performance by running this Docker image on your own machine:
+
+```bash
+docker run -p 8888:8888 fugueproject/tutorials:latest
+```
+
+
+## Jupyter Notebook Extension
+
+There is an accompanying [notebook extension](https://pypi.org/project/fugue-jupyter/) for FugueSQL that lets users use the `%%fsql` cell magic. The extension also provides syntax highlighting for FugueSQL cells. It works for both classic notebook and Jupyter Lab. More details can be found in the [installation instructions](https://github.com/fugue-project/fugue-jupyter#install).
+
+![FugueSQL gif](https://miro.medium.com/max/700/1*6091-RcrOPyifJTLjo0anA.gif)
+
+
+## Ecosystem
+
+By being an abstraction layer, Fugue can be used with a lot of other open-source projects seamlessly.
+
+Python backends:
+
+*   [Pandas](https://github.com/pandas-dev/pandas)
+*   [Polars](https://www.pola.rs) (DataFrames only)
+*   [Spark](https://github.com/apache/spark)
+*   [Dask](https://github.com/dask/dask)
+*   [Ray](http://github.com/ray-project/ray)
+*   [Ibis](https://github.com/ibis-project/ibis/)
+
+FugueSQL backends:
+
+*   Pandas - FugueSQL can run on Pandas
+*   [Duckdb](https://github.com/duckdb/duckdb) - in-process SQL OLAP database management
+*   [dask-sql](https://github.com/dask-contrib/dask-sql) - SQL interface for Dask
+*   SparkSQL
+*   [BigQuery](https://fugue-tutorials.readthedocs.io/tutorials/integrations/warehouses/bigquery.html)
+*   Trino
+
+
+Fugue is available as a backend or can integrate with the following projects:
+
+*   [WhyLogs](https://whylogs.readthedocs.io/en/latest/examples/integrations/Fugue_Profiling.html?highlight=fugue) - data profiling
+*   [PyCaret](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/pycaret.html) - low code machine learning
+*   [Nixtla](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/nixtla.html) - timeseries modelling
+*   [Prefect](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/prefect.html) - workflow orchestration
+*   [Pandera](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/pandera.html) - data validation
+*   [Datacompy (by Capital One)](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/datacompy.html) - comparing DataFrames
+
+Registered 3rd party extensions (majorly for Fugue SQL) include:
+
+*   [Pandas plot](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.plot.html) - visualize data using matplotlib or plotly
+*   [Seaborn](https://seaborn.pydata.org/api.html) - visualize data using seaborn
+*   [WhyLogs](https://whylogs.readthedocs.io/en/latest/examples/integrations/Fugue_Profiling.html?highlight=fugue) - visualize data profiling
+*   [Vizzu](https://github.com/vizzuhq/ipyvizzu) - visualize data using ipyvizzu
+
+## Community and Contributing
+
+Feel free to message us on [Slack](http://slack.fugue.ai). We also have [contributing instructions](CONTRIBUTING.md).
+
+### Case Studies
+
+*   [How LyftLearn Democratizes Distributed Compute through Kubernetes Spark and Fugue](https://eng.lyft.com/how-lyftlearn-democratizes-distributed-compute-through-kubernetes-spark-and-fugue-c0875b97c3d9)
+*   [Clobotics - Large Scale Image Processing with Spark through Fugue](https://medium.com/fugue-project/large-scale-image-processing-with-spark-through-fugue-e510b9813da8)
+*   [Architecture for a data lake REST API using Delta Lake, Fugue & Spark (article by bitsofinfo)](https://bitsofinfo.wordpress.com/2023/08/14/data-lake-rest-api-delta-lake-fugue-spark)
+
+### Mentioned Uses
+
+*   [Productionizing Data Science at Interos, Inc. (LinkedIn post by Anthony Holten)](https://www.linkedin.com/posts/anthony-holten_pandas-spark-dask-activity-7022628193983459328-QvcF)
+*   [Multiple Time Series Forecasting with Fugue & Nixtla at Bain & Company (LinkedIn post by Fahad Akbar)](https://www.linkedin.com/posts/fahadakbar_fugue-datascience-forecasting-activity-7041119034813124608-u08q?utm_source=share&utm_medium=member_desktop)
+
+## Further Resources
+
+View some of our latest conferences presentations and content. For a more complete list, check the [Content](https://fugue-tutorials.readthedocs.io/tutorials/resources/content.html) page in the tutorials.
+
+### Blogs
+
+*   [Why Pandas-like Interfaces are Sub-optimal for Distributed Computing](https://towardsdatascience.com/why-pandas-like-interfaces-are-sub-optimal-for-distributed-computing-322dacbce43)
+*   [Introducing FugueSQL — SQL for Pandas, Spark, and Dask DataFrames (Towards Data Science by Khuyen Tran)](https://towardsdatascience.com/introducing-fuguesql-sql-for-pandas-spark-and-dask-dataframes-63d461a16b27)
+
+### Conferences
+
+*   [Distributed Machine Learning at Lyft](https://www.youtube.com/watch?v=_IVyIOV0LgY)
+*   [Comparing the Different Ways to Scale Python and Pandas Code](https://www.youtube.com/watch?v=b3ae0m_XTys)
+*   [Large Scale Data Validation with Spark and Dask (PyCon US)](https://www.youtube.com/watch?v=2AdvBgjO_3Q)
+*   [FugueSQL - The Enhanced SQL Interface for Pandas, Spark, and Dask DataFrames (PyData Global)](https://www.youtube.com/watch?v=OBpnGYjNBBI)
+*   [Distributed Hybrid Parameter Tuning](https://www.youtube.com/watch?v=_GBjqskD8Qk)

{fugue-0.9.0.dev4 → fugue-0.9.2}/fugue/_utils/io.py

@@ -20,6 +20,10 @@ class FileParser(object):
         self._has_glob = "*" in path or "?" in path
         self._raw_path = path
         self._fs, self._fs_path = url_to_fs(path)
+        if not self._has_glob and self._fs.isdir(self._fs_path):
+            self._is_dir = True
+        else:
+            self._is_dir = False
         if not self.is_local:
             self._path = self._fs.unstrip_protocol(self._fs_path)
         else:
@@ -43,11 +47,15 @@ class FileParser(object):
         return self
 
     @property
-    def has_glob(self):
+    def is_dir(self) -> bool:
+        return self._is_dir
+
+    @property
+    def has_glob(self) -> bool:
         return self._has_glob
 
     @property
-    def is_local(self):
+    def is_local(self) -> bool:
         return isinstance(self._fs, LocalFileSystem)
 
     def join(self, path: str, format_hint: Optional[str] = None) -> "FileParser":
@@ -65,6 +73,10 @@ class FileParser(object):
     def path(self) -> str:
         return self._path
 
+    def as_dir_path(self) -> str:
+        assert_or_throw(self.is_dir, f"{self.raw_path} is not a directory")
+        return self.path + self._fs.sep
+
     @property
     def raw_path(self) -> str:
         return self._raw_path

{fugue-0.9.0.dev4 → fugue-0.9.2}/fugue/dataframe/function_wrapper.py

@@ -20,6 +20,7 @@ from triad.collections.function_wrapper import (
     PositionalParam,
     function_wrapper,
 )
+from triad.utils.convert import compare_annotations
 from triad.utils.iter import EmptyAwareIterable, make_empty_aware
 
 from ..constants import FUGUE_ENTRYPOINT
@@ -37,6 +38,14 @@ from .iterable_dataframe import IterableDataFrame
 from .pandas_dataframe import PandasDataFrame
 
 
+def _compare_iter(tp: Any) -> Any:
+    return lambda x: compare_annotations(
+        x, Iterable[tp]  # type:ignore
+    ) or compare_annotations(
+        x, Iterator[tp]  # type:ignore
+    )
+
+
 @function_wrapper(FUGUE_ENTRYPOINT)
 class DataFrameFunctionWrapper(FunctionWrapper):
     @property
@@ -71,6 +80,7 @@ class DataFrameFunctionWrapper(FunctionWrapper):
         p.update(kwargs)
         has_kw = False
         rargs: Dict[str, Any] = {}
+        row_param_info: Any = None
         for k, v in self._params.items():
             if isinstance(v, (PositionalParam, KeywordParam)):
                 if isinstance(v, KeywordParam):
@@ -81,7 +91,16 @@ class DataFrameFunctionWrapper(FunctionWrapper):
                         isinstance(p[k], DataFrame),
                         lambda: TypeError(f"{p[k]} is not a DataFrame"),
                     )
-                    rargs[k] = v.to_input_data(p[k], ctx=ctx)
+                    if v.is_per_row:  # pragma: no cover
+                        # TODO: this branch is used only if row annotations
+                        # are allowed as input
+                        assert_or_throw(
+                            row_param_info is None,
+                            lambda: ValueError("only one row parameter is allowed"),
+                        )
+                        row_param_info = (k, v, p[k])
+                    else:
+                        rargs[k] = v.to_input_data(p[k], ctx=ctx)
                 else:
                     rargs[k] = p[k]  # TODO: should we do auto type conversion?
                 del p[k]
@@ -91,12 +110,40 @@ class DataFrameFunctionWrapper(FunctionWrapper):
             rargs.update(p)
         elif not ignore_unknown and len(p) > 0:
             raise ValueError(f"{p} are not acceptable parameters")
+        if row_param_info is None:
+            return self._run_func(rargs, output, output_schema, ctx, raw=False)
+        else:  # pragma: no cover
+            # input contains row parameter
+            # TODO: this branch is used only if row annotations are allowed as input
+
+            def _dfs() -> Iterable[Any]:
+                k, v, df = row_param_info
+                for row in v.to_input_rows(df, ctx):
+                    rargs[k] = None
+                    _rargs = rargs.copy()
+                    _rargs[k] = row
+                    yield self._run_func(_rargs, output, output_schema, ctx, raw=True)
+
+            if not output:
+                sum(1 for _ in _dfs())
+                return
+            else:
+                return self._rt.iterable_to_output_df(_dfs(), output_schema, ctx)
+
+    def _run_func(
+        self,
+        rargs: Dict[str, Any],
+        output: bool,
+        output_schema: Any,
+        ctx: Any,
+        raw: bool,
+    ) -> Any:
         rt = self._func(**rargs)
         if not output:
             if isinstance(self._rt, _DataFrameParamBase):
                 self._rt.count(rt)
             return
-        if isinstance(self._rt, _DataFrameParamBase):
+        if not raw and isinstance(self._rt, _DataFrameParamBase):
             return self._rt.to_output_df(rt, output_schema, ctx=ctx)
         return rt
 
@@ -111,6 +158,7 @@ fugue_annotated_param = DataFrameFunctionWrapper.annotated_param
         annotation == Callable
         or annotation == callable  # pylint: disable=comparison-with-callable
         or str(annotation).startswith("typing.Callable")
+        or str(annotation).startswith("collections.abc.Callable")
     ),
 )
 class _CallableParam(AnnotatedParam):
@@ -125,6 +173,9 @@ class _CallableParam(AnnotatedParam):
         or annotation == Optional[callable]
         or str(annotation).startswith("typing.Union[typing.Callable")  # 3.8-
         or str(annotation).startswith("typing.Optional[typing.Callable")  # 3.9+
+        or str(annotation).startswith(
+            "typing.Optional[collections.abc.Callable]"
+        )  # 3.9+
     ),
 )
 class _OptionalCallableParam(AnnotatedParam):
@@ -136,14 +187,30 @@ class _DataFrameParamBase(AnnotatedParam):
         super().__init__(param)
         assert_or_throw(self.required, lambda: TypeError(f"{self} must be required"))
 
+    @property
+    def is_per_row(self) -> bool:
+        return False
+
     def to_input_data(self, df: DataFrame, ctx: Any) -> Any:  # pragma: no cover
         raise NotImplementedError
 
+    def to_input_rows(
+        self,
+        df: DataFrame,
+        ctx: Any,
+    ) -> Iterable[Any]:
+        raise NotImplementedError  # pragma: no cover
+
     def to_output_df(
         self, df: Any, schema: Any, ctx: Any
     ) -> DataFrame:  # pragma: no cover
         raise NotImplementedError
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        raise NotImplementedError
+
     def count(self, df: Any) -> int:  # pragma: no cover
         raise NotImplementedError
 
@@ -173,6 +240,36 @@ class DataFrameParam(_DataFrameParamBase):
             return sum(1 for _ in df.as_array_iterable())
 
 
+@fugue_annotated_param(DataFrame, "r", child_can_reuse_code=True)
+class RowParam(_DataFrameParamBase):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
+    @property
+    def is_per_row(self) -> bool:
+        return True
+
+    def count(self, df: Any) -> int:
+        return 1
+
+
+@fugue_annotated_param(Dict[str, Any])
+class DictParam(RowParam):  # pragma: no cover
+    # TODO: this class is used only if row annotations are allowed as input
+    def to_input_rows(self, df: DataFrame, ctx: Any) -> Iterable[Any]:
+        yield from df.as_dict_iterable()
+
+    def to_output_df(self, output: Dict[str, Any], schema: Any, ctx: Any) -> DataFrame:
+        return ArrayDataFrame([list(output.values())], schema)
+
+    def iterable_to_output_df(
+        self, dfs: Iterable[Dict[str, Any]], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        params: Dict[str, Any] = {}
+        if schema is not None:
+            params["schema"] = Schema(schema).pa_schema
+        adf = pa.Table.from_pylist(list(dfs), **params)
+        return ArrowDataFrame(adf)
+
+
 @fugue_annotated_param(AnyDataFrame)
 class _AnyDataFrameParam(DataFrameParam):
     def to_output_df(self, output: AnyDataFrame, schema: Any, ctx: Any) -> DataFrame:
@@ -198,6 +295,15 @@ class LocalDataFrameParam(DataFrameParam):
         )
         return output
 
+    def iterable_to_output_df(
+        self, dfs: Iterable[Any], schema: Any, ctx: Any
+    ) -> DataFrame:  # pragma: no cover
+        def _dfs() -> Iterable[DataFrame]:
+            for df in dfs:
+                yield self.to_output_df(df, schema, ctx)
+
+        return LocalDataFrameIterableDataFrame(_dfs(), schema=schema)
+
     def count(self, df: LocalDataFrame) -> int:
         if df.is_bounded:
             return df.count()
@@ -228,10 +334,7 @@ class _ListListParam(_LocalNoSchemaDataFrameParam):
         return len(df)
 
 
-@fugue_annotated_param(
-    Iterable[List[Any]],
-    matcher=lambda x: x == Iterable[List[Any]] or x == Iterator[List[Any]],
-)
+@fugue_annotated_param(Iterable[List[Any]], matcher=_compare_iter(List[Any]))
 class _IterableListParam(_LocalNoSchemaDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[List[Any]]:
@@ -288,10 +391,7 @@ class _ListDictParam(_LocalNoSchemaDataFrameParam):
         return len(df)
 
 
-@fugue_annotated_param(
-    Iterable[Dict[str, Any]],
-    matcher=lambda x: x == Iterable[Dict[str, Any]] or x == Iterator[Dict[str, Any]],
-)
+@fugue_annotated_param(Iterable[Dict[str, Any]], matcher=_compare_iter(Dict[str, Any]))
 class _IterableDictParam(_LocalNoSchemaDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[Dict[str, Any]]:
@@ -360,10 +460,7 @@ class _PandasParam(LocalDataFrameParam):
         return "pandas"
 
 
-@fugue_annotated_param(
-    Iterable[pd.DataFrame],
-    matcher=lambda x: x == Iterable[pd.DataFrame] or x == Iterator[pd.DataFrame],
-)
+@fugue_annotated_param(Iterable[pd.DataFrame], matcher=_compare_iter(pd.DataFrame))
 class _IterablePandasParam(LocalDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[pd.DataFrame]:
@@ -419,10 +516,7 @@ class _PyArrowTableParam(LocalDataFrameParam):
         return "pyarrow"
 
 
-@fugue_annotated_param(
-    Iterable[pa.Table],
-    matcher=lambda x: x == Iterable[pa.Table] or x == Iterator[pa.Table],
-)
+@fugue_annotated_param(Iterable[pa.Table], matcher=_compare_iter(pa.Table))
 class _IterableArrowParam(LocalDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> Iterable[pa.Table]:

{fugue-0.9.0.dev4 → fugue-0.9.2}/fugue/extensions/transformer/convert.py

@@ -375,7 +375,7 @@ class _FuncAsTransformer(Transformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspq]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspqr]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -410,7 +410,7 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^[lspq][fF]?x*z?$", "^[lspnq]$"
+            func, "^[lspq][fF]?x*z?$", "^[lspnqr]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
@@ -503,7 +503,7 @@ class _FuncAsCoTransformer(CoTransformer):
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = schema  # type: ignore
@@ -562,7 +562,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
 
         tr = _FuncAsOutputCoTransformer()
         tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
-            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$"
+            func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnqr]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
         tr._output_schema_arg = None  # type: ignore