fugue 0.8.5.dev1__py3-none-any.whl → 0.8.6.dev2__py3-none-any.whl

fugue/dataframe/dataframe.py

@@ -113,6 +113,13 @@ class DataFrame(Dataset):
     def as_pandas(self) -> pd.DataFrame:
         """Convert to pandas DataFrame"""
         pdf = pd.DataFrame(self.as_array(), columns=self.columns)
+        if len(pdf) == 0:  # TODO: move to triad
+            return pd.DataFrame(
+                {
+                    k: pd.Series(dtype=v.type.to_pandas_dtype())
+                    for k, v in self.schema.items()
+                }
+            )
         return PD_UTILS.enforce_type(pdf, self.schema.pa_schema, null_safe=True)
 
     def as_arrow(self, type_safe: bool = False) -> pa.Table:

fugue/dataframe/dataframe_iterable_dataframe.py

@@ -165,7 +165,7 @@ class LocalDataFrameIterableDataFrame(LocalUnboundedDataFrame):
 
     def as_pandas(self) -> pd.DataFrame:
         if self.empty:
-            return ArrayDataFrame([], self.schema).as_pandas()
+            return PandasDataFrame(schema=self.schema).as_pandas()
 
         return pd.concat(df.as_pandas() for df in self.native)
 

fugue/dataframe/function_wrapper.py

@@ -23,9 +23,10 @@ from triad.collections.function_wrapper import (
 from triad.utils.iter import EmptyAwareIterable, make_empty_aware
 
 from ..constants import FUGUE_ENTRYPOINT
+from ..dataset.api import count as df_count
 from .array_dataframe import ArrayDataFrame
 from .arrow_dataframe import ArrowDataFrame
-from .dataframe import DataFrame, LocalDataFrame
+from .dataframe import AnyDataFrame, DataFrame, LocalDataFrame, as_fugue_df
 from .dataframe_iterable_dataframe import (
     IterableArrowDataFrame,
     IterablePandasDataFrame,
@@ -172,6 +173,19 @@ class DataFrameParam(_DataFrameParamBase):
             return sum(1 for _ in df.as_array_iterable())
 
 
+@fugue_annotated_param(AnyDataFrame)
+class _AnyDataFrameParam(DataFrameParam):
+    def to_output_df(self, output: AnyDataFrame, schema: Any, ctx: Any) -> DataFrame:
+        return (
+            as_fugue_df(output)
+            if schema is None
+            else as_fugue_df(output, schema=schema)
+        )
+
+    def count(self, df: Any) -> int:
+        return df_count(df)
+
+
 @fugue_annotated_param(LocalDataFrame, "l", child_can_reuse_code=True)
 class LocalDataFrameParam(DataFrameParam):
     def to_input_data(self, df: DataFrame, ctx: Any) -> LocalDataFrame:
@@ -333,6 +347,9 @@ class _PandasParam(LocalDataFrameParam):
 
     @no_type_check
     def to_output_df(self, output: pd.DataFrame, schema: Any, ctx: Any) -> DataFrame:
+        _schema: Optional[Schema] = None if schema is None else Schema(schema)
+        if _schema is not None and _schema.names != list(output.columns):
+            output = output[_schema.names]
         return PandasDataFrame(output, schema)
 
     @no_type_check
@@ -361,8 +378,15 @@ class _IterablePandasParam(LocalDataFrameParam):
         self, output: Iterable[pd.DataFrame], schema: Any, ctx: Any
     ) -> DataFrame:
         def dfs():
+            _schema: Optional[Schema] = None if schema is None else Schema(schema)
+            has_return = False
             for df in output:
-                yield PandasDataFrame(df, schema)
+                if _schema is not None and _schema.names != list(df.columns):
+                    df = df[_schema.names]
+                yield PandasDataFrame(df, _schema)
+                has_return = True
+            if not has_return and _schema is not None:
+                yield PandasDataFrame(schema=_schema)
 
         return IterablePandasDataFrame(dfs())
 
@@ -381,7 +405,12 @@ class _PyArrowTableParam(LocalDataFrameParam):
 
     def to_output_df(self, output: Any, schema: Any, ctx: Any) -> DataFrame:
         assert isinstance(output, pa.Table)
-        return ArrowDataFrame(output, schema=schema)
+        adf: DataFrame = ArrowDataFrame(output)
+        if schema is not None:
+            _schema = Schema(schema)
+            if adf.schema != _schema:
+                adf = adf[_schema.names].alter_columns(_schema)
+        return adf
 
     def count(self, df: Any) -> int:  # pragma: no cover
         return df.count()
@@ -409,13 +438,15 @@ class _IterableArrowParam(LocalDataFrameParam):
     ) -> DataFrame:
         def dfs():
             _schema: Optional[Schema] = None if schema is None else Schema(schema)
+            has_return = False
             for df in output:
-                adf = ArrowDataFrame(df)
-                if _schema is not None and not (  # pylint: disable-all
-                    adf.schema == schema
-                ):
+                adf: DataFrame = ArrowDataFrame(df)
+                if _schema is not None and adf.schema != _schema:
                     adf = adf[_schema.names].alter_columns(_schema)
                 yield adf
+                has_return = True
+            if not has_return and _schema is not None:
+                yield ArrowDataFrame(schema=_schema)
 
         return IterableArrowDataFrame(dfs())
 

{fugue-0.8.5.dev1.dist-info → fugue-0.8.6.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.5.dev1
+Version: 0.8.6.dev2
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
-Requires-Dist: triad (>=0.9.0)
+Requires-Dist: triad (>=0.9.1)
 Requires-Dist: adagio (>=0.2.4)
 Requires-Dist: pyarrow (>=0.15.1)
 Requires-Dist: pandas (>=1.2.0)
-Requires-Dist: qpd (>=0.4.3)
+Requires-Dist: qpd (>=0.4.4)
 Requires-Dist: fugue-sql-antlr (>=0.1.6)
 Requires-Dist: sqlglot
 Requires-Dist: jinja2
@@ -33,8 +33,8 @@ Requires-Dist: sqlglot ; extra == 'all'
 Requires-Dist: jinja2 ; extra == 'all'
 Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.6) ; extra == 'all'
 Requires-Dist: pyspark (>=3.1.1) ; extra == 'all'
-Requires-Dist: ray[data] (>=2.0.0) ; extra == 'all'
-Requires-Dist: qpd[dask] (>=0.4.3) ; extra == 'all'
+Requires-Dist: ray[data] (>=2.1.0) ; extra == 'all'
+Requires-Dist: qpd[dask] (>=0.4.4) ; extra == 'all'
 Requires-Dist: notebook ; extra == 'all'
 Requires-Dist: jupyterlab ; extra == 'all'
 Requires-Dist: ipython (>=7.10.0) ; extra == 'all'
@@ -43,21 +43,21 @@ Requires-Dist: pyarrow (>=6.0.1) ; extra == 'all'
 Requires-Dist: polars ; extra == 'all'
 Requires-Dist: dask[dataframe,distributed] ; (python_version < "3.8") and extra == 'all'
 Requires-Dist: ibis-framework (>=2.1.1) ; (python_version < "3.8") and extra == 'all'
-Requires-Dist: dask[dataframe,distributed] (>=2022.9.0) ; (python_version >= "3.8") and extra == 'all'
-Requires-Dist: ibis-framework (>=3.2.0) ; (python_version >= "3.8") and extra == 'all'
+Requires-Dist: dask[dataframe,distributed] (<2023.7.1,>=2022.9.0) ; (python_version >= "3.8") and extra == 'all'
+Requires-Dist: ibis-framework (<6,>=3.2.0) ; (python_version >= "3.8") and extra == 'all'
 Provides-Extra: cpp_sql_parser
 Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.6) ; extra == 'cpp_sql_parser'
 Provides-Extra: dask
-Requires-Dist: qpd[dask] (>=0.4.3) ; extra == 'dask'
+Requires-Dist: qpd[dask] (>=0.4.4) ; extra == 'dask'
 Requires-Dist: dask[dataframe,distributed] ; (python_version < "3.8") and extra == 'dask'
-Requires-Dist: dask[dataframe,distributed] (>=2022.9.0) ; (python_version >= "3.8") and extra == 'dask'
+Requires-Dist: dask[dataframe,distributed] (<2023.7.1,>=2022.9.0) ; (python_version >= "3.8") and extra == 'dask'
 Provides-Extra: duckdb
 Requires-Dist: duckdb (>=0.5.0) ; extra == 'duckdb'
 Requires-Dist: pyarrow (>=6.0.1) ; extra == 'duckdb'
 Requires-Dist: numpy ; extra == 'duckdb'
 Provides-Extra: ibis
 Requires-Dist: ibis-framework (>=2.1.1) ; (python_version < "3.8") and extra == 'ibis'
-Requires-Dist: ibis-framework (>=3.2.0) ; (python_version >= "3.8") and extra == 'ibis'
+Requires-Dist: ibis-framework (<6,>=3.2.0) ; (python_version >= "3.8") and extra == 'ibis'
 Provides-Extra: notebook
 Requires-Dist: notebook ; extra == 'notebook'
 Requires-Dist: jupyterlab ; extra == 'notebook'
@@ -65,13 +65,13 @@ Requires-Dist: ipython (>=7.10.0) ; extra == 'notebook'
 Provides-Extra: polars
 Requires-Dist: polars ; extra == 'polars'
 Provides-Extra: ray
-Requires-Dist: ray[data] (>=2.0.0) ; extra == 'ray'
+Requires-Dist: ray[data] (>=2.1.0) ; extra == 'ray'
 Requires-Dist: duckdb (>=0.5.0) ; extra == 'ray'
 Requires-Dist: pyarrow (>=6.0.1) ; extra == 'ray'
 Provides-Extra: spark
 Requires-Dist: pyspark (>=3.1.1) ; extra == 'spark'
 Provides-Extra: sql
-Requires-Dist: qpd (>=0.4.3) ; extra == 'sql'
+Requires-Dist: qpd (>=0.4.4) ; extra == 'sql'
 Requires-Dist: fugue-sql-antlr (>=0.1.6) ; extra == 'sql'
 Requires-Dist: sqlglot ; extra == 'sql'
 Requires-Dist: jinja2 ; extra == 'sql'
@@ -222,9 +222,16 @@ Fugue can be installed through pip or conda. For example:
 pip install fugue
 ```
 
+In order to use Fugue SQL, it is strongly recommended to install the `sql` extra:
+
+```bash
+pip install fugue[sql]
+```
+
 It also has the following installation extras:
 
-*   **spark**: to support Spark as the [ExecutionEngine](https://fugue-tutorials.readthedocs.io/tutorials/advanced/execution_engine.html)
+*   **sql**: to support Fugue SQL. Without this extra, the non-SQL part still works. Before Fugue 0.9.0, this extra is included in Fugue's core dependency so you don't need to install explicitly. **But for 0,9.0+, this becomes required if you want to use Fugue SQL.**
+*   **spark**: to support Spark as the [ExecutionEngine](https://fugue-tutorials.readthedocs.io/tutorials/advanced/execution_engine.html).
 *   **dask**: to support Dask as the ExecutionEngine.
 *   **ray**: to support Ray as the ExecutionEngine.
 *   **duckdb**: to support DuckDB as the ExecutionEngine, read [details](https://fugue-tutorials.readthedocs.io/tutorials/integrations/backends/duckdb.html).

{fugue-0.8.5.dev1.dist-info → fugue-0.8.6.dev2.dist-info}/RECORD

@@ -4,6 +4,7 @@ fugue/constants.py,sha256=crd0VqX8WtBcjSUNwZDi2LDIEkhUMWOlSn73H8JI9ds,3385
 fugue/dev.py,sha256=GQCkezBBl4V0lVDWhGtUQKqomiCxgR9dMhfqj9C8cS8,1369
 fugue/exceptions.py,sha256=ylP8gkZL8ao_ZLinNYKv16FPyO_n7c29dN-4QChUxi0,1544
 fugue/plugins.py,sha256=SJ-jqs04StHIHJ65lgdGP0IDopVIGBDpmzHHllNK8p0,998
+fugue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/registry.py,sha256=SNULGv08f37fRO-cIxFDmnVcod7ref2fNLSK6G7nVnI,868
 fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/_utils/display.py,sha256=JV8oDA7efHm1wceZulCBOY5dMvjbWHvIm6ASisKfoWY,3164
@@ -27,10 +28,10 @@ fugue/dataframe/__init__.py,sha256=zm7TbsaJLIvfm7zymWm2LGcuJd3nxfGsFnQiyrSnenM,6
 fugue/dataframe/api.py,sha256=c5Err3c-ayl-k28IUi6kV_ClDWX30NpVNkv97hQKDac,9862
 fugue/dataframe/array_dataframe.py,sha256=oBfN545NTGdYJ5zPIRv7hXRR-R_OW1JieyOfnl296oU,4447
 fugue/dataframe/arrow_dataframe.py,sha256=h0DJH8G0MEgfkyvX4U957iqDXIgvTtrP7YED5iEjizI,12098
-fugue/dataframe/dataframe.py,sha256=rIjaOplyRDt_BfpFwZWMjZALDqa03NePs9tFiLSW6Jg,17247
-fugue/dataframe/dataframe_iterable_dataframe.py,sha256=0gvb12D0s4VntNZ-M8J_Pic2XlFpw7upXf4hfMn2ufY,7255
+fugue/dataframe/dataframe.py,sha256=XOn9x_aKWOLdpach7RHxg2PoP-hpfsyjoRFvLx6xKV0,17496
+fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=DjyIrNBj5Bv9AwIl2I2fG5ClcKe3OoW_eBkyEabYY5Y,13505
+fugue/dataframe/function_wrapper.py,sha256=r6H1SQWaag2eSbJ50327t_bt7MZunbOMOl9OcOcQW2E,14827
 fugue/dataframe/iterable_dataframe.py,sha256=Kn5HZnVU4o1nn9mbbQxaV8rGG869wImZcOCK3AdlA-M,4627
 fugue/dataframe/pandas_dataframe.py,sha256=ZWqI-ZUFiSP7giJ3siRlrZcMedI_fyuoLn227H0YRvw,10453
 fugue/dataframe/utils.py,sha256=nQVU01jspB1NSeRiagE71uzRibDqvyGwi94ZfHwNHD0,10508
@@ -103,7 +104,7 @@ fugue_ibis/__init__.py,sha256=PcUt66KlLyGGicad7asq5j2U567_fhR0HzvWQBhV1VM,362
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
 fugue_ibis/dataframe.py,sha256=Y4Wn--oAlCvmqStY92AgUaAKqr9l6jSfJ2EXOhDFk9M,7302
-fugue_ibis/execution_engine.py,sha256=igZ_rHflORPKp2h8HMSlzuv50P3ZaaijsvWyc8gilow,18540
+fugue_ibis/execution_engine.py,sha256=p5zy0IBXiJgLi67RBHCRcHgZsaJMANdNSpUxz0k_6C0,18453
 fugue_ibis/extensions.py,sha256=H8l-SPfoqLuUoILtOuL2nccOpoL83zHeSoIhoqjtWQM,6905
 fugue_ibis/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ibis/execution/ibis_engine.py,sha256=-HdPnIFWD83n5WITdzJiu4attH7GOcO041wkT5Y5ChA,1499
@@ -120,17 +121,17 @@ fugue_polars/polars_dataframe.py,sha256=Ll4ZUuRhAETWtmSf87KsdUCqZPiexFqy4FiPkvWQ
 fugue_polars/registry.py,sha256=gd6qQ-OxYtTAQFyvYbLDPXmSvCR-LW6n5K5ylgMY_7A,2950
 fugue_ray/__init__.py,sha256=HzEHfG2mpc0ugf3nf1Pdy15Bhg35K6maZpYejn1aoyI,119
 fugue_ray/_constants.py,sha256=vu5l1w-Wi-2V_nm0HLXKOYhh5HdWRCc5yQktO2XzhOg,569
-fugue_ray/dataframe.py,sha256=shUtnQbAquN2s5bR3Rx1QUGxxLz_g-Az9O0QDcXaCD0,10377
+fugue_ray/dataframe.py,sha256=u6X9OvCGZta6cjHvQ-CcK4xckXNvKHK3EEMHn8dFrHM,10577
 fugue_ray/execution_engine.py,sha256=PUj1Fgqsg-6DDFG9KNip7NenTX_bfYvBjcFjmXp8LNo,12596
 fugue_ray/registry.py,sha256=xJRAhbwNrg695EwghQDnVtTKi4YkqZ0_61BD4OAblSA,1685
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
-fugue_ray/_utils/dataframe.py,sha256=xaw9Pbfjsnd_0vmbGRJJbAPUPWMZaYptw4LfDdIgvHo,3145
+fugue_ray/_utils/dataframe.py,sha256=gUgdmDws8gFm7YjYyISRDgRx_-ksxtQu6oSpwIGkxfM,4457
 fugue_ray/_utils/io.py,sha256=gHfx70tdXPHmVL6nHxVhmCO5KpWjjyDG8qKT1Lbpav4,8737
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=xoM2-SwVRFfSyfEEnx4g4b0GO6XgN-DQLvXtUrAMq1Q,9510
-fugue_spark/execution_engine.py,sha256=n_ZdQI4ysbgdDsjyWwbwhyBsgcg_icc9MAmVyWEn2RQ,32029
+fugue_spark/execution_engine.py,sha256=nssgfqt2h1OjGlK5iuGEyF-lt2pEokmsjrqo6K4C1Kg,32440
 fugue_spark/ibis_engine.py,sha256=Yl5xxwROo1idcD2hFaylaI1IpmBUgbvOZRWtcrE0Zjo,1697
 fugue_spark/registry.py,sha256=kyIMk6dAiKRSKCHawQKyXu9DhZ24T6j3gL57TiOAZ8c,4162
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -142,14 +143,14 @@ fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=piY_bVtLylQWrZbencrQwT89IXgvgBmNscddJ6vZieY,76338
+fugue_test/builtin_suite.py,sha256=sT_Btm7cpSsuLnt_PpNFeLZVc2WPmF5NQy4GKVri_-c,78396
 fugue_test/dataframe_suite.py,sha256=6uM7_et2Y55-ePIssG9G_K9mXBYNjPXnpunuCh0xKhw,19082
 fugue_test/execution_suite.py,sha256=HzM_7jUhnt1kD2jGaaPhyr-Q_vtRV4nCQTaLtMnzTuU,50948
 fugue_test/ibis_suite.py,sha256=Dk4AHVD00RcFsNm9VvJ4_4LOyFdGX30OnAtpO2SPruE,3529
-fugue_version/__init__.py,sha256=K0kGrhh1kzVisZcoSkeuJdC06rTwxufV05Vy2hOVGoo,22
-fugue-0.8.5.dev1.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.8.5.dev1.dist-info/METADATA,sha256=DA5pNJvD69RYFX-B8waTg-SCXrwzAVXj0zKPTUzajps,17870
-fugue-0.8.5.dev1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-fugue-0.8.5.dev1.dist-info/entry_points.txt,sha256=By9wWIZDuUtDISL9QiORS0_BzCDtYK7s_MZLtFuloFA,374
-fugue-0.8.5.dev1.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.8.5.dev1.dist-info/RECORD,,
+fugue_version/__init__.py,sha256=VpASnrti7EGWxUfSWGgERUfe7NLJltfVXYosOzHbpPg,22
+fugue-0.8.6.dev2.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.8.6.dev2.dist-info/METADATA,sha256=pGSp1pOq2BHvHd3KiRL3YzFHbNKzmKDMUPd8Hcs__xc,18284
+fugue-0.8.6.dev2.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
+fugue-0.8.6.dev2.dist-info/entry_points.txt,sha256=Ta1DD9RIgS_YfhieUvM6PgAzuOYmhlnKe9fWbns9sLc,374
+fugue-0.8.6.dev2.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.8.6.dev2.dist-info/RECORD,,

{fugue-0.8.5.dev1.dist-info → fugue-0.8.6.dev2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.41.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{fugue-0.8.5.dev1.dist-info → fugue-0.8.6.dev2.dist-info}/entry_points.txt

@@ -1,11 +1,11 @@
 [fugue.plugins]
 dask = fugue_dask.registry [dask]
-dask_ibis = fugue_dask.ibis_engine [ibis,dask]
+dask_ibis = fugue_dask.ibis_engine [dask,ibis]
 duckdb = fugue_duckdb.registry [duckdb]
-duckdb_ibis = fugue_duckdb.ibis_engine [ibis,duckdb]
+duckdb_ibis = fugue_duckdb.ibis_engine [duckdb,ibis]
 ibis = fugue_ibis [ibis]
 polars = fugue_polars.registry [polars]
 ray = fugue_ray.registry [ray]
 spark = fugue_spark.registry [spark]
-spark_ibis = fugue_spark.ibis_engine [spark,ibis]
+spark_ibis = fugue_spark.ibis_engine [ibis,spark]
 

fugue_ibis/execution_engine.py

@@ -92,20 +92,19 @@ class IbisSQLEngine(SQLEngine):
         _df2 = self.to_df(df2)
         key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
         on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
+        if ibis.__version__ < "6":
+            suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
+        else:  # pragma: no cover
+            # breaking change in ibis 6.0
+            suffixes = dict(lname="", rname=_JOIN_RIGHT_SUFFIX)
         if how.lower() == "cross":
-            tb = _df1.native.cross_join(_df2.native, suffixes=("", _JOIN_RIGHT_SUFFIX))
+            tb = _df1.native.cross_join(_df2.native, **suffixes)
         elif how.lower() == "right_outer":
-            tb = _df2.native.left_join(
-                _df1.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df2.native.left_join(_df1.native, on_fields, **suffixes)
         elif how.lower() == "left_outer":
-            tb = _df1.native.left_join(
-                _df2.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df1.native.left_join(_df2.native, on_fields, **suffixes)
         elif how.lower() == "full_outer":
-            tb = _df1.native.outer_join(
-                _df2.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df1.native.outer_join(_df2.native, on_fields, **suffixes)
             cols: List[Any] = []
             for k in end_schema.names:
                 if k not in key_schema:
@@ -116,17 +115,11 @@ class IbisSQLEngine(SQLEngine):
                     )
             tb = tb[cols]
         elif how.lower() in ["semi", "left_semi"]:
-            tb = _df1.native.semi_join(
-                _df2.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df1.native.semi_join(_df2.native, on_fields, **suffixes)
         elif how.lower() in ["anti", "left_anti"]:
-            tb = _df1.native.anti_join(
-                _df2.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df1.native.anti_join(_df2.native, on_fields, **suffixes)
         else:
-            tb = _df1.native.inner_join(
-                _df2.native, on_fields, suffixes=("", _JOIN_RIGHT_SUFFIX)
-            )
+            tb = _df1.native.inner_join(_df2.native, on_fields, **suffixes)
         return self.to_df(tb[end_schema.names], schema=end_schema)
 
     def union(self, df1: DataFrame, df2: DataFrame, distinct: bool = True) -> DataFrame:

fugue_ray/_utils/dataframe.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import pandas as pd
 import pyarrow as pa
+import ray
 import ray.data as rd
 from triad import Schema
 
@@ -13,15 +14,51 @@ from .._constants import _ZERO_COPY
 _RAY_NULL_REPR = "__RAY_NULL__"
 
 
-def get_dataset_format(df: rd.Dataset) -> Optional[str]:
-    df.fully_executed()
+def is_materialized(df: rd.Dataset) -> bool:
+    if hasattr(rd.dataset, "MaterializedDataset"):
+        return isinstance(df, rd.dataset.MaterializedDataset)
+    return df.is_fully_executed()  # pragma: no cover
+
+
+def materialize(df: rd.Dataset) -> rd.Dataset:
+    if not is_materialized(df):
+        if hasattr(df, "materialize"):
+            df = df.materialize()
+        else:  # pragma: no cover
+            df = df.fully_executed()
+    return df
+
+
+def get_dataset_format(df: rd.Dataset) -> Tuple[Optional[str], rd.Dataset]:
+    df = materialize(df)
     if df.count() == 0:
-        return None
-    if hasattr(df, "_dataset_format"):  # pragma: no cover
-        return df._dataset_format()  # ray<2.2
-    ctx = rd.context.DatasetContext.get_current()
-    ctx.use_streaming_executor = False
-    return df.dataset_format()  # ray>=2.2
+        return None, df
+    if ray.__version__ < "2.5.0":  # pragma: no cover
+        if hasattr(df, "_dataset_format"):  # pragma: no cover
+            return df._dataset_format(), df  # ray<2.2
+        ctx = rd.context.DatasetContext.get_current()
+        ctx.use_streaming_executor = False
+        return df.dataset_format(), df  # ray>=2.2
+    else:
+        schema = df.schema(fetch_if_missing=True)
+        if schema is None:  # pragma: no cover
+            return None, df
+        if isinstance(schema.base_schema, pa.Schema):
+            return "arrow", df
+        return "pandas", df
+
+
+def to_schema(schema: Any) -> Schema:  # pragma: no cover
+    if isinstance(schema, pa.Schema):
+        return Schema(schema)
+    if ray.__version__ >= "2.5.0":
+        if isinstance(schema, rd.Schema):
+            if hasattr(schema, "base_schema") and isinstance(
+                schema.base_schema, pa.Schema
+            ):
+                return Schema(schema.base_schema)
+            return Schema(list(zip(schema.names, schema.types)))
+    raise ValueError(f"{schema} is not supported")
 
 
 def build_empty(schema: Schema) -> rd.Dataset:

fugue_ray/dataframe.py

@@ -18,7 +18,7 @@ from fugue.plugins import (
 )
 
 from ._constants import _ZERO_COPY
-from ._utils.dataframe import build_empty, get_dataset_format
+from ._utils.dataframe import build_empty, get_dataset_format, materialize, to_schema
 
 
 class RayDataFrame(DataFrame):
@@ -52,7 +52,7 @@ class RayDataFrame(DataFrame):
             self._native = build_empty(schema)
             return
         if isinstance(df, rd.Dataset):
-            fmt = get_dataset_format(df)
+            fmt, df = get_dataset_format(df)
             if fmt is None:  # empty:
                 schema = _input_schema(schema).assert_not_empty()
                 super().__init__(schema)
@@ -62,7 +62,7 @@ class RayDataFrame(DataFrame):
                 rdf = rd.from_arrow_refs(df.to_arrow_refs())
             elif fmt == "arrow":
                 rdf = df
-            else:
+            else:  # pragma: no cover
                 raise NotImplementedError(
                     f"Ray Dataset in {fmt} format is not supported"
                 )
@@ -156,8 +156,7 @@ class RayDataFrame(DataFrame):
 
     def persist(self, **kwargs: Any) -> "RayDataFrame":
         # TODO: it mutates the dataframe, is this a good bahavior
-        if not self.native.is_fully_executed():  # pragma: no cover
-            self.native.fully_executed()
+        self._native = materialize(self._native)
         return self
 
     def count(self) -> int:
@@ -226,11 +225,12 @@ class RayDataFrame(DataFrame):
     ) -> Tuple[rd.Dataset, Schema]:
         if internal_schema:
             return rdf, schema
-        if get_dataset_format(rdf) is None:  # empty
+        fmt, rdf = get_dataset_format(rdf)
+        if fmt is None:  # empty
             schema = _input_schema(schema).assert_not_empty()
             return build_empty(schema), schema
-        if schema is None or schema == rdf.schema(fetch_if_missing=True):
-            return rdf, rdf.schema(fetch_if_missing=True)
+        if schema is None or schema == to_schema(rdf.schema(fetch_if_missing=True)):
+            return rdf, to_schema(rdf.schema(fetch_if_missing=True))
 
         def _alter(table: pa.Table) -> pa.Table:  # pragma: no cover
             return ArrowDataFrame(table).alter_columns(schema).native  # type: ignore
@@ -263,12 +263,15 @@ def _rd_as_local(df: rd.Dataset) -> bool:
 
 @get_column_names.candidate(lambda df: isinstance(df, rd.Dataset))
 def _get_ray_dataframe_columns(df: rd.Dataset) -> List[Any]:
-    fmt = get_dataset_format(df)
-    if fmt == "pandas":
-        return list(df.schema(True).names)
-    elif fmt == "arrow":
-        return [f.name for f in df.schema(True)]
-    raise NotImplementedError(f"{fmt} is not supported")  # pragma: no cover
+    if hasattr(df, "columns"):  # higher version of ray
+        return df.columns(fetch_if_missing=True)
+    else:  # pragma: no cover
+        fmt, _ = get_dataset_format(df)
+        if fmt == "pandas":
+            return list(df.schema(True).names)
+        elif fmt == "arrow":
+            return df.schema(fetch_if_missing=True).names
+        raise NotImplementedError(f"{fmt} is not supported")  # pragma: no cover
 
 
 @rename.candidate(lambda df, *args, **kwargs: isinstance(df, rd.Dataset))

fugue_spark/execution_engine.py

@@ -134,9 +134,8 @@ class SparkMapEngine(MapEngine):
     ) -> DataFrame:
         output_schema = Schema(output_schema)
         if self._should_use_pandas_udf(output_schema):
-            # pandas udf can only be used for pyspark > 3
             if len(partition_spec.partition_by) > 0:
-                if partition_spec.algo == "coarse":
+                if partition_spec.algo in ["coarse", "even"]:
                     return self._map_by_pandas_udf(
                         df,
                         map_func=map_func,
@@ -145,7 +144,18 @@ class SparkMapEngine(MapEngine):
                         on_init=on_init,
                         map_func_format_hint=map_func_format_hint,
                     )
-                elif partition_spec.algo != "even" or self.is_spark_connect:
+                else:
+                    if (  # not simple partitioning
+                        partition_spec.algo != "hash"
+                        or partition_spec.num_partitions != "0"
+                    ):
+                        # TODO: not sure if presort should be done
+                        # on physical partition level
+                        df = self.to_df(
+                            self.execution_engine.repartition(
+                                df, PartitionSpec(partition_spec, presort=[])
+                            )
+                        )
                     return self._group_map_by_pandas_udf(
                         df,
                         map_func=map_func,
@@ -154,7 +164,7 @@ class SparkMapEngine(MapEngine):
                         on_init=on_init,
                         map_func_format_hint=map_func_format_hint,
                     )
-            elif len(partition_spec.partition_by) == 0:
+            else:
                 return self._map_by_pandas_udf(
                     df,
                     map_func=map_func,

fugue_test/builtin_suite.py

@@ -22,6 +22,7 @@ from triad import SerializableRLock
 
 import fugue.api as fa
 from fugue import (
+    AnyDataFrame,
     ArrayDataFrame,
     CoTransformer,
     DataFrame,
@@ -365,6 +366,12 @@ class BuiltInTests(object):
                 dag.output(dict(df=a), using=mock_outputter2)
                 a.partition(num=3).output(MockOutputter3)
                 dag.output(dict(aa=a, bb=b), using=MockOutputter4)
+
+                a = dag.create(mock_creator2, params=dict(p=2))
+                b = dag.create(mock_creator2, params=dict(p=2))
+                c = dag.process(a, b, using=mock_processor4)
+                c.assert_eq(ArrayDataFrame([[2]], "a:int"))
+                dag.output(a, b, using=mock_outputter4)
             dag.run(self.engine)
 
         def test_zip(self):
@@ -435,9 +442,14 @@ class BuiltInTests(object):
             # this test is important for using mapInPandas in spark
 
             # schema: *,c:int
-            def mt_pandas(dfs: Iterable[pd.DataFrame]) -> Iterator[pd.DataFrame]:
+            def mt_pandas(
+                dfs: Iterable[pd.DataFrame], empty: bool = False
+            ) -> Iterator[pd.DataFrame]:
                 for df in dfs:
-                    yield df.assign(c=2)
+                    if not empty:
+                        df = df.assign(c=2)
+                        df = df[reversed(list(df.columns))]
+                        yield df
 
             with FugueWorkflow() as dag:
                 a = dag.df([[1, 2], [3, 4]], "a:int,b:int")
@@ -445,10 +457,25 @@ class BuiltInTests(object):
                 dag.df([[1, 2, 2], [3, 4, 2]], "a:int,b:int,c:int").assert_eq(b)
             dag.run(self.engine)
 
+            # when iterable returns nothing
+            with FugueWorkflow() as dag:
+                a = dag.df([[1, 2], [3, 4]], "a:int,b:int")
+                # without partitioning
+                b = a.transform(mt_pandas, params=dict(empty=True))
+                dag.df([], "a:int,b:int,c:int").assert_eq(b)
+                # with partitioning
+                b = a.partition_by("a").transform(mt_pandas, params=dict(empty=True))
+                dag.df([], "a:int,b:int,c:int").assert_eq(b)
+            dag.run(self.engine)
+
             # schema: *
-            def mt_arrow(dfs: Iterable[pa.Table]) -> Iterator[pa.Table]:
+            def mt_arrow(
+                dfs: Iterable[pa.Table], empty: bool = False
+            ) -> Iterator[pa.Table]:
                 for df in dfs:
-                    yield df
+                    if not empty:
+                        df = df.select(reversed(df.schema.names))
+                        yield df
 
             # schema: a:long
             def mt_arrow_2(dfs: Iterable[pa.Table]) -> Iterator[pa.Table]:
@@ -463,6 +490,17 @@ class BuiltInTests(object):
                 dag.df([[1], [3]], "a:long").assert_eq(b)
             dag.run(self.engine)
 
+            # when iterable returns nothing
+            with FugueWorkflow() as dag:
+                a = dag.df([[1, 2], [3, 4]], "a:int,b:int")
+                # without partitioning
+                b = a.transform(mt_arrow, params=dict(empty=True))
+                dag.df([], "a:int,b:int").assert_eq(b)
+                # with partitioning
+                b = a.partition_by("a").transform(mt_arrow, params=dict(empty=True))
+                dag.df([], "a:int,b:int").assert_eq(b)
+            dag.run(self.engine)
+
         def test_transform_binary(self):
             with FugueWorkflow() as dag:
                 a = dag.df([[1, pickle.dumps([0, "a"])]], "a:int,b:bytes")
@@ -1829,6 +1867,10 @@ def mock_creator(p: int) -> DataFrame:
     return ArrayDataFrame([[p]], "a:int")
 
 
+def mock_creator2(p: int) -> AnyDataFrame:
+    return fa.as_fugue_df([[p]], schema="a:int")
+
+
 def mock_processor(df1: List[List[Any]], df2: List[List[Any]]) -> DataFrame:
     return ArrayDataFrame([[len(df1) + len(df2)]], "a:int")
 
@@ -1844,6 +1886,10 @@ class MockProcessor3(Processor):
         return ArrayDataFrame([[sum(s.count() for s in dfs.values())]], "a:int")
 
 
+def mock_processor4(df1: AnyDataFrame, df2: AnyDataFrame) -> AnyDataFrame:
+    return ArrayDataFrame([[fa.count(df1) + fa.count(df2)]], "a:int")
+
+
 def mock_outputter(df1: List[List[Any]], df2: List[List[Any]]) -> None:
     assert len(df1) == len(df2)
 
@@ -1857,6 +1903,10 @@ class MockOutputter3(Outputter):
         assert "3" == self.partition_spec.num_partitions
 
 
+def mock_outputter4(df1: AnyDataFrame, df2: AnyDataFrame) -> None:
+    assert fa.count(df1) == fa.count(df2)
+
+
 class MockOutputter4(Outputter):
     def process(self, dfs):
         for k, v in dfs.items():
@@ -1895,8 +1945,8 @@ def mock_tf0(df: pd.DataFrame, p=1, col="p") -> pd.DataFrame:
 
 # schema: *,ct:int,p:int
 def mock_tf1(df: pd.DataFrame, p=1) -> pd.DataFrame:
-    df["ct"] = df.shape[0]
     df["p"] = p
+    df["ct"] = df.shape[0]
     return df
 
 

fugue_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.5"
+__version__ = "0.8.6"