fugue 0.8.7.dev5__py3-none-any.whl → 0.8.7.dev6__py3-none-any.whl

fugue_dask/dataframe.py

@@ -3,20 +3,21 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
 import dask.dataframe as dd
 import pandas as pd
 import pyarrow as pa
+from triad import assert_or_throw
 from triad.collections.schema import Schema
 from triad.utils.assertion import assert_arg_not_none
 from triad.utils.pandas_like import PD_UTILS
 from triad.utils.pyarrow import cast_pa_table
 
-from fugue.dataframe import (
-    ArrowDataFrame,
-    DataFrame,
-    LocalBoundedDataFrame,
-    PandasDataFrame,
-)
+from fugue.dataframe import DataFrame, LocalBoundedDataFrame, PandasDataFrame
 from fugue.dataframe.dataframe import _input_schema
+from fugue.dataframe.pandas_dataframe import _pd_as_dicts
 from fugue.exceptions import FugueDataFrameOperationError
 from fugue.plugins import (
+    as_array,
+    as_array_iterable,
+    as_dict_iterable,
+    as_dicts,
     as_local_bounded,
     count,
     drop_columns,
@@ -32,7 +33,7 @@ from fugue.plugins import (
 )
 
 from ._constants import FUGUE_DASK_USE_ARROW
-from ._utils import DASK_UTILS, get_default_partitions
+from ._utils import DASK_UTILS, collect, get_default_partitions
 
 
 class DaskDataFrame(DataFrame):
@@ -150,8 +151,16 @@ class DaskDataFrame(DataFrame):
         )
 
     def as_arrow(self, type_safe: bool = False) -> pa.Table:
-        adf = pa.Table.from_pandas(self.native.compute().reset_index(drop=True))
-        return cast_pa_table(adf, self.schema.pa_schema)
+        schema = self.schema.pa_schema
+        return pa.concat_tables(
+            collect(
+                self.native,
+                lambda df: cast_pa_table(
+                    pa.Table.from_pandas(df.reset_index(drop=True), schema=schema),
+                    schema=schema,
+                ),
+            )
+        )
 
     def rename(self, columns: Dict[str, str]) -> DataFrame:
         try:
@@ -170,17 +179,28 @@ class DaskDataFrame(DataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        df: DataFrame = self
-        if columns is not None:
-            df = df[columns]
-        return ArrowDataFrame(df.as_pandas(), schema=df.schema).as_array(
-            type_safe=type_safe
-        )
+        chunks = _to_array_chunks(self.native, columns, type_safe, schema=self.schema)
+        res: List[List[Any]] = []
+        for x in chunks:
+            res += x
+        return res
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        yield from self.as_array(columns=columns, type_safe=type_safe)
+        chunks = _to_array_chunks(self.native, columns, type_safe, schema=self.schema)
+        for x in chunks:
+            yield from x
+
+    def as_dicts(
+        self, columns: Optional[List[str]] = None, type_safe: bool = False
+    ) -> List[Dict[str, Any]]:
+        return _dd_as_dicts(self.native, columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None, type_safe: bool = False
+    ) -> Iterable[Dict[str, Any]]:
+        yield from _dd_as_dict_iterable(self.native, columns)
 
     def head(
         self, n: int, columns: Optional[List[str]] = None
@@ -197,8 +217,11 @@ class DaskDataFrame(DataFrame):
             assert_arg_not_none(schema, "schema")
             return pdf, schema
         DASK_UTILS.ensure_compatible(pdf)
-        pschema = Schema(DASK_UTILS.to_schema(pdf))
-        if schema is None or pschema == schema:
+        # when pdf contains bytes, or any object types, and schema contains str
+        # there is no way to get the real schema of the pdf, (pschema will contain
+        # strs instead of the real types) so we have to force cast it to the schema
+        if schema is None:
+            pschema = Schema(DASK_UTILS.to_schema(pdf))
             return pdf, pschema.assert_not_empty()
         pdf = pdf[schema.assert_not_empty().names]
         return (
@@ -295,6 +318,48 @@ def _dd_head(
     return PandasDataFrame(res) if as_fugue else res
 
 
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_array(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    chunks = _to_array_chunks(df, columns, type_safe)
+    res: List[List[Any]] = []
+    for x in chunks:
+        res += x
+    return res
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_array_iterable(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    chunks = _to_array_chunks(df, columns, type_safe)
+    for x in chunks:
+        yield from x
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_dicts(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Dict[str, Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+    res: List[Dict[str, Any]] = []
+    for x in collect(_df, lambda df: _pd_as_dicts(df, columns)):
+        res += x
+    return res
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_dict_iterable(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Dict[str, Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+    for x in collect(_df, lambda df: _pd_as_dicts(df, columns)):
+        yield from x
+
+
 def _assert_no_missing(df: dd.DataFrame, columns: Iterable[Any]) -> None:
     missing = set(columns) - set(df.columns)
     if len(missing) > 0:
@@ -303,3 +368,25 @@ def _assert_no_missing(df: dd.DataFrame, columns: Iterable[Any]) -> None:
 
 def _adjust_df(res: dd.DataFrame, as_fugue: bool):
     return res if not as_fugue else DaskDataFrame(res)
+
+
+def _to_array_chunks(
+    df: dd.DataFrame,
+    columns: Optional[List[str]] = None,
+    type_safe: bool = False,
+    schema: Optional[Schema] = None,
+) -> Tuple[List[Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+
+    def _to_list(pdf: pd.DataFrame) -> List[Any]:
+        return list(
+            PD_UTILS.as_array_iterable(
+                pdf,
+                schema=None if schema is None else schema.pa_schema,
+                columns=columns,
+                type_safe=type_safe,
+            )
+        )
+
+    return collect(_df, _to_list)

fugue_duckdb/dataframe.py

@@ -3,21 +3,33 @@ from typing import Any, Dict, Iterable, List, Optional
 import pandas as pd
 import pyarrow as pa
 from duckdb import DuckDBPyRelation
-from triad import Schema
+from triad import Schema, assert_or_throw
 from triad.utils.pyarrow import LARGE_TYPES_REPLACEMENT, replace_types_in_table
 
-from fugue import ArrayDataFrame, ArrowDataFrame, DataFrame, LocalBoundedDataFrame
+from fugue import ArrowDataFrame, DataFrame, LocalBoundedDataFrame
 from fugue.dataframe.arrow_dataframe import _pa_table_as_pandas
+from fugue.dataframe.utils import (
+    pa_table_as_array,
+    pa_table_as_array_iterable,
+    pa_table_as_dict_iterable,
+    pa_table_as_dicts,
+)
 from fugue.exceptions import FugueDataFrameOperationError, FugueDatasetEmptyError
 from fugue.plugins import (
+    as_array,
+    as_array_iterable,
     as_arrow,
+    as_dict_iterable,
+    as_dicts,
     as_fugue_dataset,
     as_local_bounded,
     as_pandas,
+    drop_columns,
     get_column_names,
     get_num_partitions,
     get_schema,
     is_df,
+    select_columns,
 )
 
 from ._utils import encode_column_name, to_duck_type, to_pa_type
@@ -59,13 +71,10 @@ class DuckDataFrame(LocalBoundedDataFrame):
         return len(self._rel)
 
     def _drop_cols(self, cols: List[str]) -> DataFrame:
-        cols = [col for col in self._rel.columns if col not in cols]
-        rel = self._rel.project(",".join(encode_column_name(n) for n in cols))
-        return DuckDataFrame(rel)
+        return DuckDataFrame(_drop_duckdb_columns(self._rel, cols))
 
     def _select_cols(self, keys: List[Any]) -> DataFrame:
-        rel = self._rel.project(",".join(encode_column_name(n) for n in keys))
-        return DuckDataFrame(rel)
+        return DuckDataFrame(_select_duckdb_columns(self._rel, keys))
 
     def rename(self, columns: Dict[str, str]) -> DataFrame:
         _assert_no_missing(self._rel, columns.keys())
@@ -109,38 +118,29 @@ class DuckDataFrame(LocalBoundedDataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        if columns is not None:
-            return self[columns].as_array(type_safe=type_safe)
-        return self._fetchall(self._rel)
+        return _duck_as_array(self._rel, columns=columns, type_safe=type_safe)
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        if columns is not None:
-            yield from self[columns].as_array_iterable(type_safe=type_safe)
-        else:
-            yield from self._fetchall(self._rel)
+        yield from _duck_as_array_iterable(
+            self._rel, columns=columns, type_safe=type_safe
+        )
+
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return _duck_as_dicts(self._rel, columns=columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        yield from _duck_as_dict_iterable(self._rel, columns=columns)
 
     def head(
         self, n: int, columns: Optional[List[str]] = None
     ) -> LocalBoundedDataFrame:
         if columns is not None:
             return self[columns].head(n)
-        return ArrayDataFrame(self._fetchall(self._rel.limit(n)), schema=self.schema)
-
-    def _fetchall(self, rel: DuckDBPyRelation) -> List[List[Any]]:
-        map_pos = [i for i, t in enumerate(self.schema.types) if pa.types.is_map(t)]
-        if len(map_pos) == 0:
-            return [list(x) for x in rel.fetchall()]
-        else:
-
-            def to_list(row: Any) -> List[Any]:
-                res = list(row)
-                for p in map_pos:
-                    res[p] = list(zip(row[p]["key"], row[p]["value"]))
-                return res
-
-            return [to_list(x) for x in rel.fetchall()]
+        return ArrowDataFrame(_duck_as_arrow(self._rel.limit(n)))
 
 
 @as_fugue_dataset.candidate(lambda df, **kwargs: isinstance(df, DuckDBPyRelation))
@@ -186,6 +186,64 @@ def _get_duckdb_columns(df: DuckDBPyRelation) -> List[Any]:
     return list(df.columns)
 
 
+@select_columns.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _select_duckdb_columns(
+    df: DuckDBPyRelation, columns: List[Any]
+) -> DuckDBPyRelation:
+    if len(columns) == 0:
+        raise FugueDataFrameOperationError("must select at least one column")
+    _assert_no_missing(df, columns)
+    return df.project(",".join(encode_column_name(n) for n in columns))
+
+
+@drop_columns.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _drop_duckdb_columns(df: DuckDBPyRelation, columns: List[str]) -> DuckDBPyRelation:
+    # if len(columns) == 0:
+    #   return df
+    _columns = {c: 1 for c in columns}
+    cols = [col for col in df.columns if _columns.pop(col, None) is None]
+    assert_or_throw(
+        len(cols) > 0, FugueDataFrameOperationError("must keep at least one column")
+    )
+    assert_or_throw(
+        len(_columns) == 0,
+        FugueDataFrameOperationError("found nonexistent columns {_columns}"),
+    )
+    return df.project(",".join(encode_column_name(n) for n in cols))
+
+
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _duck_as_array(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    return pa_table_as_array(df.arrow(), columns=columns)
+
+
+@as_array_iterable.candidate(
+    lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation)
+)
+def _duck_as_array_iterable(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    yield from pa_table_as_array_iterable(df.arrow(), columns=columns)
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _duck_as_dicts(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    return pa_table_as_dicts(df.arrow(), columns=columns)
+
+
+@as_dict_iterable.candidate(
+    lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation)
+)
+def _duck_as_dict_iterable(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    yield from pa_table_as_dict_iterable(df.arrow(), columns=columns)
+
+
 def _assert_no_missing(df: DuckDBPyRelation, columns: Iterable[Any]) -> None:
     missing = set(columns) - set(df.columns)
     if len(missing) > 0:

fugue_ibis/dataframe.py

@@ -143,6 +143,19 @@ class IbisDataFrame(DataFrame):
                 type_safe=type_safe
             )
 
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        if columns is not None:
+            return self[columns].as_dicts()
+        return self.as_local().as_dicts()
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        if columns is not None:
+            yield from self[columns].as_dict_iterable()
+        else:
+            yield from self._to_iterable_df(self._table).as_dict_iterable()
+
     def head(
         self, n: int, columns: Optional[List[str]] = None
     ) -> LocalBoundedDataFrame:

fugue_polars/polars_dataframe.py

@@ -14,22 +14,32 @@ from triad.utils.pyarrow import (
 
 from fugue import ArrowDataFrame
 from fugue.api import (
+    as_array,
+    as_array_iterable,
     as_arrow,
+    as_dict_iterable,
+    as_dicts,
     drop_columns,
     get_column_names,
     get_schema,
     is_df,
+    is_empty,
     rename,
     select_columns,
 )
 from fugue.dataframe.dataframe import DataFrame, LocalBoundedDataFrame, _input_schema
+from fugue.dataframe.utils import (
+    pa_table_as_array,
+    pa_table_as_array_iterable,
+    pa_table_as_dict_iterable,
+    pa_table_as_dicts,
+)
 from fugue.dataset.api import (
     as_local,
     as_local_bounded,
     count,
     get_num_partitions,
     is_bounded,
-    is_empty,
     is_local,
 )
 from fugue.exceptions import FugueDataFrameOperationError
@@ -52,7 +62,7 @@ class PolarsDataFrame(LocalBoundedDataFrame):
     ):
         if df is None:
             schema = _input_schema(schema).assert_not_empty()
-            self._native: pa.Table = build_empty_pl(schema)
+            self._native: pl.DataFrame = build_empty_pl(schema)
             super().__init__(schema)
             return
         else:
@@ -73,7 +83,7 @@ class PolarsDataFrame(LocalBoundedDataFrame):
 
     @property
     def empty(self) -> bool:
-        return self._native.shape[0] == 0
+        return self._native.is_empty()
 
     def peek_array(self) -> List[Any]:
         self.assert_not_empty()
@@ -118,26 +128,20 @@ class PolarsDataFrame(LocalBoundedDataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        tdf = self.native
-        if columns is not None:
-            tdf = tdf.select(columns)
-        return [list(row) for row in tdf.rows()]
+        return _pl_as_array(self.native, columns=columns)
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        if not self.empty:
-            yield from ArrowDataFrame(_pl_as_arrow(self.native)).as_array_iterable(
-                columns=columns
-            )
+        yield from _pl_as_array_iterable(self.native, columns=columns)
+
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return _pl_as_dicts(self.native, columns=columns)
 
     def as_dict_iterable(
         self, columns: Optional[List[str]] = None
     ) -> Iterable[Dict[str, Any]]:
-        if not self.empty:
-            yield from ArrowDataFrame(_pl_as_arrow(self.native)).as_dict_iterable(
-                columns=columns
-            )
+        yield from _pl_as_dict_iterable(self.native, columns=columns)
 
 
 @as_local.candidate(lambda df: isinstance(df, pl.DataFrame))
@@ -174,7 +178,7 @@ def _pl_is_bounded(df: pl.DataFrame) -> bool:
 
 @is_empty.candidate(lambda df: isinstance(df, pl.DataFrame))
 def _pl_is_empty(df: pl.DataFrame) -> bool:
-    return df.shape[0] == 0
+    return df.is_empty()
 
 
 @is_local.candidate(lambda df: isinstance(df, pl.DataFrame))
@@ -228,6 +232,39 @@ def _select_pa_columns(df: pl.DataFrame, columns: List[Any]) -> pl.DataFrame:
     return df.select(columns)
 
 
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, pl.DataFrame))
+def _pl_as_array(
+    df: pl.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[List[Any]]:
+    _df = df if columns is None else _select_pa_columns(df, columns)
+    adf = _pl_as_arrow(_df)
+    return pa_table_as_array(adf, columns=columns)
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pl.DataFrame))
+def _pl_as_array_iterable(
+    df: pl.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[List[Any]]:
+    _df = df if columns is None else _select_pa_columns(df, columns)
+    yield from pa_table_as_array_iterable(_df.to_arrow(), columns=columns)
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, pl.DataFrame))
+def _pl_as_dicts(
+    df: pl.DataFrame, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    _df = df if columns is None else _select_pa_columns(df, columns)
+    return pa_table_as_dicts(_df.to_arrow(), columns=columns)
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pl.DataFrame))
+def _pl_as_dict_iterable(
+    df: pl.DataFrame, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    _df = df if columns is None else _select_pa_columns(df, columns)
+    yield from pa_table_as_dict_iterable(_df.to_arrow(), columns=columns)
+
+
 def _assert_no_missing(df: pl.DataFrame, columns: Iterable[Any]) -> None:
     missing = [x for x in columns if x not in df.schema.keys()]
     if len(missing) > 0:

fugue_ray/dataframe.py

@@ -4,14 +4,22 @@ import pandas as pd
 import pyarrow as pa
 import ray
 import ray.data as rd
+from triad import assert_or_throw
 from triad.collections.schema import Schema
 from triad.utils.pyarrow import cast_pa_table
 
 from fugue.dataframe import ArrowDataFrame, DataFrame, LocalBoundedDataFrame
 from fugue.dataframe.dataframe import _input_schema
+from fugue.dataframe.utils import pa_table_as_array, pa_table_as_dicts
 from fugue.exceptions import FugueDataFrameOperationError, FugueDatasetEmptyError
 from fugue.plugins import (
+    as_array,
+    as_array_iterable,
+    as_arrow,
+    as_dict_iterable,
+    as_dicts,
     as_local_bounded,
+    as_pandas,
     get_column_names,
     get_num_partitions,
     is_df,
@@ -141,13 +149,11 @@ class RayDataFrame(DataFrame):
     def _select_cols(self, cols: List[Any]) -> DataFrame:
         if cols == self.columns:
             return self
-        rdf = self.native.map_batches(
-            lambda b: b.select(cols),
-            batch_format="pyarrow",
-            **_ZERO_COPY,
-            **self._remote_args(),
+        return RayDataFrame(
+            self.native.select_columns(cols),
+            self.schema.extract(cols),
+            internal_schema=True,
         )
-        return RayDataFrame(rdf, self.schema.extract(cols), internal_schema=True)
 
     def peek_array(self) -> List[Any]:
         data = self.native.limit(1).to_pandas().values.tolist()
@@ -164,10 +170,10 @@ class RayDataFrame(DataFrame):
         return self.native.count()
 
     def as_arrow(self, type_safe: bool = False) -> pa.Table:
-        return pa.concat_tables(_get_arrow_tables(self.native))
+        return _rd_as_arrow(self.native)
 
     def as_pandas(self) -> pd.DataFrame:
-        return self.as_arrow().to_pandas()
+        return _rd_as_pandas(self.native)
 
     def rename(self, columns: Dict[str, str]) -> DataFrame:
         try:
@@ -201,18 +207,20 @@ class RayDataFrame(DataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        df: DataFrame = self
-        if columns is not None:
-            df = df[columns]
-        adf = df.as_arrow()
-        if adf.shape[0] == 0:
-            return []
-        return ArrowDataFrame(adf).as_array(type_safe=type_safe)
+        return _rd_as_array(self.native, columns, type_safe)
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        yield from self.as_array(columns=columns, type_safe=type_safe)
+        yield from _rd_as_array_iterable(self.native, columns, type_safe)
+
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return _rd_as_dicts(self.native, columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        yield from _rd_as_dict_iterable(self.native, columns)
 
     def head(
         self, n: int, columns: Optional[List[str]] = None
@@ -259,8 +267,8 @@ def _rd_num_partitions(df: rd.Dataset) -> int:
 
 
 @as_local_bounded.candidate(lambda df: isinstance(df, rd.Dataset))
-def _rd_as_local(df: rd.Dataset) -> bool:
-    return pa.concat_tables(_get_arrow_tables(df))
+def _rd_as_local(df: rd.Dataset) -> pa.Table:
+    return _rd_as_arrow(df)
 
 
 @get_column_names.candidate(lambda df: isinstance(df, rd.Dataset))
@@ -290,10 +298,54 @@ def _rename_ray_dataframe(df: rd.Dataset, columns: Dict[str, Any]) -> rd.Dataset
     )
 
 
+@as_pandas.candidate(lambda df: isinstance(df, rd.Dataset))
+def _rd_as_pandas(df: rd.Dataset) -> pd.DataFrame:
+    return _rd_as_arrow(df).to_pandas()
+
+
+@as_arrow.candidate(lambda df: isinstance(df, rd.Dataset))
+def _rd_as_arrow(df: rd.Dataset) -> pa.Table:
+    return pa.concat_tables(_get_arrow_tables(df))
+
+
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, rd.Dataset))
+def _rd_as_array(
+    df: rd.Dataset, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df.select_columns(columns)
+    adf = _rd_as_arrow(_df)
+    return pa_table_as_array(adf)
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, rd.Dataset))
+def _rd_as_array_iterable(
+    df: rd.Dataset, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    yield from _rd_as_array(df, columns, type_safe)
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, rd.Dataset))
+def _rd_as_dicts(
+    df: rd.Dataset, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Dict[str, Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df.select_columns(columns)
+    adf = _rd_as_arrow(_df)
+    return pa_table_as_dicts(adf)
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, rd.Dataset))
+def _rd_as_dict_iterable(
+    df: rd.Dataset, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Dict[str, Any]]:
+    yield from _rd_as_dicts(df, columns, type_safe)
+
+
 def _get_arrow_tables(df: rd.Dataset) -> Iterable[pa.Table]:
     last_empty: Any = None
     empty = True
-    for block in df.get_internal_block_refs():
+    for block in df.to_arrow_refs():
         tb = ray.get(block)
         if tb.shape[0] > 0:
             yield tb