fugue 0.8.7.dev4__py3-none-any.whl → 0.8.7.dev6__py3-none-any.whl

fugue/api.py

@@ -6,6 +6,7 @@ from .dataframe.api import (
     as_array_iterable,
     as_arrow,
     as_dict_iterable,
+    as_dicts,
     as_fugue_df,
     as_pandas,
     drop_columns,

fugue/dataframe/api.py

@@ -11,12 +11,12 @@ from .dataframe import AnyDataFrame, DataFrame, as_fugue_df
 
 @fugue_plugin
 def is_df(df: Any) -> bool:
-    """Whether ``df`` is a DataFrame like object"""
+    """Whether the input object is any type of DataFrame"""
     return isinstance(df, DataFrame)
 
 
 def get_native_as_df(df: AnyDataFrame) -> AnyDataFrame:
-    """Return the dataframe form of the input ``df``.
+    """Return the dataframe form of any dataframe.
     If ``df`` is a :class:`~.DataFrame`, then call the
     :meth:`~.DataFrame.native_as_df`, otherwise, it depends on whether there is
     a correspondent function handling it.
@@ -30,30 +30,49 @@ def get_native_as_df(df: AnyDataFrame) -> AnyDataFrame:
 
 @fugue_plugin
 def get_schema(df: AnyDataFrame) -> Schema:
-    """Get the schema of the ``df``
+    """The generic function to get the schema of any dataframe
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :return: the Schema object
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            import fugue.api as fa
+            import pandas as pd
+
+            df = pd.DataFrame([[0,1],[2,3]], columns=["a","b"])
+            fa.get_schema(df) # == Schema("a:long,b:long")
+
+    .. related_topics
+        How to get schema of any dataframe using Fugue?
     """
     return as_fugue_df(df).schema
 
 
 @fugue_plugin
 def as_pandas(df: AnyDataFrame) -> pd.DataFrame:
-    """Convert ``df`` to a Pandas DataFrame
+    """The generic function to convert any dataframe to a Pandas DataFrame
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :return: the Pandas DataFrame
+
+    .. related_topics
+        How to convert any dataframe to a pandas dataframe?
     """
     return as_fugue_df(df).as_pandas()
 
 
 @fugue_plugin
 def as_arrow(df: AnyDataFrame) -> pa.Table:
-    """Convert ``df`` to a PyArrow Table
+    """The generic function to convert any dataframe to a PyArrow Table
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :return: the PyArrow Table
+
+    .. related_topics
+        How to convert any dataframe to a pyarrow dataframe?
     """
     return as_fugue_df(df).as_arrow()
 
@@ -62,7 +81,7 @@ def as_arrow(df: AnyDataFrame) -> pa.Table:
 def as_array(
     df: AnyDataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> List[Any]:  # pragma: no cover
-    """Convert df to 2-dimensional native python array
+    """The generic function to convert any dataframe to a 2-dimensional python array
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: columns to extract, defaults to None
@@ -81,7 +100,7 @@ def as_array(
 def as_array_iterable(
     df: AnyDataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
 ) -> Iterable[Any]:  # pragma: no cover
-    """Convert df to iterable of native python arrays
+    """The generic function to convert any dataframe to iterable of python arrays
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: columns to extract, defaults to None
@@ -97,15 +116,32 @@ def as_array_iterable(
     return as_fugue_df(df).as_array_iterable(columns=columns, type_safe=type_safe)
 
 
+@fugue_plugin
+def as_dicts(
+    df: AnyDataFrame, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    """Convert any dataframe to a list of python dicts
+
+    :param df: the object that can be recognized as a dataframe by Fugue
+    :param columns: columns to extract, defaults to None
+    :return: a list of python dicts
+
+    .. note::
+
+        The default implementation enforces ``type_safe`` True
+    """
+    return as_fugue_df(df).as_dicts(columns=columns)
+
+
 @fugue_plugin
 def as_dict_iterable(
     df: AnyDataFrame, columns: Optional[List[str]] = None
 ) -> Iterable[Dict[str, Any]]:
-    """Convert df to iterable of native python dicts
+    """Convert any dataframe to iterable of python dicts
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: columns to extract, defaults to None
-    :return: iterable of native python dicts
+    :return: iterable of python dicts
 
     .. note::
 
@@ -116,7 +152,7 @@ def as_dict_iterable(
 
 @fugue_plugin
 def peek_array(df: AnyDataFrame) -> List[Any]:
-    """Peek the first row of the dataframe as an array
+    """Peek the first row of any dataframe as an array
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :return: the first row as an array
@@ -126,7 +162,7 @@ def peek_array(df: AnyDataFrame) -> List[Any]:
 
 @fugue_plugin
 def peek_dict(df: AnyDataFrame) -> Dict[str, Any]:
-    """Peek the first row of the dataframe as a array
+    """Peek the first row of any dataframe as a array
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :return: the first row as a dict
@@ -141,7 +177,7 @@ def head(
     columns: Optional[List[str]] = None,
     as_fugue: bool = False,
 ) -> AnyDataFrame:
-    """Get first n rows of the dataframe as a new local bounded dataframe
+    """Get first n rows of any dataframe as a new local bounded dataframe
 
     :param n: number of rows
     :param columns: selected columns, defaults to None (all columns)
@@ -160,7 +196,7 @@ def head(
 def alter_columns(
     df: AnyDataFrame, columns: Any, as_fugue: bool = False
 ) -> AnyDataFrame:
-    """Change column types
+    """Change column data types of any dataframe
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: |SchemaLikeObject|,
@@ -178,7 +214,7 @@ def alter_columns(
 def drop_columns(
     df: AnyDataFrame, columns: List[str], as_fugue: bool = False
 ) -> AnyDataFrame:
-    """Drop certain columns and return a new dataframe
+    """Drop certain columns of any dataframe
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: columns to drop
@@ -194,7 +230,7 @@ def drop_columns(
 def select_columns(
     df: AnyDataFrame, columns: List[Any], as_fugue: bool = False
 ) -> AnyDataFrame:
-    """Select certain columns and return a new dataframe
+    """Select certain columns of any dataframe and return a new dataframe
 
     :param df: the object that can be recognized as a dataframe by Fugue
     :param columns: columns to return

fugue/dataframe/arrow_dataframe.py

@@ -21,6 +21,10 @@ from fugue.exceptions import FugueDataFrameOperationError
 
 from .api import (
     alter_columns,
+    as_array,
+    as_array_iterable,
+    as_dict_iterable,
+    as_dicts,
     as_pandas,
     drop_columns,
     get_column_names,
@@ -30,6 +34,12 @@ from .api import (
     select_columns,
 )
 from .dataframe import DataFrame, LocalBoundedDataFrame, _input_schema
+from .utils import (
+    pa_table_as_array,
+    pa_table_as_array_iterable,
+    pa_table_as_dict_iterable,
+    pa_table_as_dicts,
+)
 
 
 class ArrowDataFrame(LocalBoundedDataFrame):
@@ -174,21 +184,20 @@ class ArrowDataFrame(LocalBoundedDataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        return list(self.as_array_iterable(columns, type_safe=type_safe))
+        return pa_table_as_array(self.native, columns=columns)
+
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return pa_table_as_dicts(self.native, columns=columns)
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        if self.empty:
-            return
-        if columns is not None:
-            for x in self[columns].as_array_iterable(type_safe=type_safe):
-                yield x
-        else:
-            d = self.native.to_pydict()
-            cols = [d[n] for n in self.columns]
-            for arr in zip(*cols):
-                yield list(arr)
+        yield from pa_table_as_array_iterable(self.native, columns=columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        yield from pa_table_as_dict_iterable(self.native, columns=columns)
 
 
 @as_local.candidate(lambda df: isinstance(df, pa.Table))
@@ -212,6 +221,34 @@ def _pa_table_as_pandas(df: pa.Table) -> pd.DataFrame:
     )
 
 
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, pa.Table))
+def _pa_table_as_array(
+    df: pa.Table, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    return pa_table_as_array(df, columns=columns)
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pa.Table))
+def _pa_table_as_array_iterable(
+    df: pa.Table, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    yield from pa_table_as_array_iterable(df, columns=columns)
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, pa.Table))
+def _pa_table_as_dicts(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    return pa_table_as_dicts(df, columns=columns)
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pa.Table))
+def _pa_table_as_dict_iterable(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    yield from pa_table_as_dict_iterable(df, columns=columns)
+
+
 @alter_columns.candidate(lambda df, *args, **kwargs: isinstance(df, pa.Table))
 def _pa_table_alter_columns(
     df: pa.Table, columns: Any, as_fugue: bool = False

fugue/dataframe/dataframe.py

@@ -237,13 +237,31 @@ class DataFrame(Dataset):
         """
         raise NotImplementedError
 
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """Convert to a list of python dicts
+
+        :param columns: columns to extract, defaults to None
+        :return: a list of python dicts
+
+        .. note::
+
+            The default implementation enforces ``type_safe`` True
+        """
+        if columns is None:
+            columns = self.columns
+        idx = range(len(columns))
+        return [
+            {columns[i]: x[i] for i in idx}
+            for x in self.as_array(columns, type_safe=True)
+        ]
+
     def as_dict_iterable(
         self, columns: Optional[List[str]] = None
     ) -> Iterable[Dict[str, Any]]:
-        """Convert to iterable of native python dicts
+        """Convert to iterable of python dicts
 
         :param columns: columns to extract, defaults to None
-        :return: iterable of native python dicts
+        :return: iterable of python dicts
 
         .. note::
 

fugue/dataframe/function_wrapper.py

@@ -269,7 +269,7 @@ class _EmptyAwareIterableListParam(_LocalNoSchemaDataFrameParam):
 class _ListDictParam(_LocalNoSchemaDataFrameParam):
     @no_type_check
     def to_input_data(self, df: DataFrame, ctx: Any) -> List[Dict[str, Any]]:
-        return list(df.as_local().as_dict_iterable())
+        return df.as_local().as_dicts()
 
     @no_type_check
     def to_output_df(

fugue/dataframe/iterable_dataframe.py

@@ -105,6 +105,9 @@ class IterableDataFrame(LocalUnboundedDataFrame):
     ) -> List[Any]:
         return list(self.as_array_iterable(columns, type_safe=type_safe))
 
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return list(self.as_dict_iterable(columns))
+
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:

fugue/dataframe/pandas_dataframe.py

@@ -1,8 +1,11 @@
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 import pandas as pd
+import pyarrow as pa
+from triad import assert_or_throw
 from triad.collections.schema import Schema
 from triad.utils.pandas_like import PD_UTILS
+from triad.utils.pyarrow import pa_batch_to_dicts
 
 from fugue.dataset.api import (
     as_fugue_dataset,
@@ -17,6 +20,10 @@ from fugue.dataset.api import (
 from fugue.exceptions import FugueDataFrameOperationError
 
 from .api import (
+    as_array,
+    as_array_iterable,
+    as_dict_iterable,
+    as_dicts,
     drop_columns,
     get_column_names,
     get_schema,
@@ -134,6 +141,9 @@ class PandasDataFrame(LocalBoundedDataFrame):
             return self
         return PandasDataFrame(self.native, new_schema)
 
+    def as_arrow(self, type_safe: bool = False) -> pa.Table:
+        return PD_UTILS.as_arrow(self.native, schema=self.schema.pa_schema)
+
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
@@ -150,6 +160,18 @@ class PandasDataFrame(LocalBoundedDataFrame):
         ):
             yield row
 
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        res: List[Dict[str, Any]] = []
+        for block in _to_dicts(self.native, columns, self.schema):
+            res += block
+        return res
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        for block in _to_dicts(self.native, columns, self.schema):
+            yield from block
+
     def head(
         self, n: int, columns: Optional[List[str]] = None
     ) -> LocalBoundedDataFrame:
@@ -272,6 +294,43 @@ def _pd_head(
     return _adjust_df(df.head(n), as_fugue=as_fugue)
 
 
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, pd.DataFrame))
+def _pd_as_array(
+    df: pd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    return list(_pd_as_array_iterable(df, columns, type_safe=type_safe))
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pd.DataFrame))
+def _pd_as_array_iterable(
+    df: pd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    for row in PD_UTILS.as_array_iterable(
+        df,
+        columns=columns,
+        type_safe=type_safe,
+    ):
+        yield row
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, pd.DataFrame))
+def _pd_as_dicts(
+    df: pd.DataFrame, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    res: List[Dict[str, Any]] = []
+    for block in _to_dicts(df, columns):
+        res += block
+    return res
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, pd.DataFrame))
+def _pd_as_dict_iterable(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    for block in _to_dicts(df, columns):
+        yield from block
+
+
 def _adjust_df(res: pd.DataFrame, as_fugue: bool):
     return res if not as_fugue else PandasDataFrame(res)
 
@@ -280,3 +339,17 @@ def _assert_no_missing(df: pd.DataFrame, columns: Iterable[Any]) -> None:
     missing = [x for x in columns if x not in df.columns]
     if len(missing) > 0:
         raise FugueDataFrameOperationError("found nonexistent columns: {missing}")
+
+
+def _to_dicts(
+    df: pd.DataFrame,
+    columns: Optional[List[str]] = None,
+    schema: Optional[Schema] = None,
+) -> Iterable[List[Dict[str, Any]]]:
+    cols = list(df.columns) if columns is None else columns
+    assert_or_throw(len(cols) > 0, ValueError("columns cannot be empty"))
+    pa_schema = schema.extract(cols).pa_schema if schema is not None else None
+    adf = PD_UTILS.as_arrow(df[cols], schema=pa_schema)
+    for batch in adf.to_batches():
+        if batch.num_rows > 0:
+            yield pa_batch_to_dicts(batch)

fugue/dataframe/utils.py

@@ -1,15 +1,16 @@
 import os
 import pickle
-from typing import Any, Iterable, Optional, Tuple
+from typing import Any, Iterable, Optional, Tuple, List, Dict
 
 import pandas as pd
 import pyarrow as pa
 from fs import open_fs
-from triad import FileSystem, Schema
+from triad import FileSystem, Schema, assert_or_throw
 from triad.collections.schema import SchemaError
 from triad.exceptions import InvalidOperationError
 from triad.utils.assertion import assert_arg_not_none
 from triad.utils.assertion import assert_or_throw as aot
+from triad.utils.pyarrow import pa_batch_to_dicts
 
 from .api import as_fugue_df, get_column_names, normalize_column_names, rename
 from .dataframe import DataFrame, LocalBoundedDataFrame
@@ -82,17 +83,19 @@ def _df_eq(
         ), f"schema mismatch {df.schema.pa_schema}, {df2.schema.pa_schema}"
         if not check_content:
             return True
+        cols: Any = df1.columns
         if no_pandas:
             dd1 = [[x.__repr__()] for x in df1.as_array_iterable(type_safe=True)]
             dd2 = [[x.__repr__()] for x in df2.as_array_iterable(type_safe=True)]
             d1 = pd.DataFrame(dd1, columns=["data"])
             d2 = pd.DataFrame(dd2, columns=["data"])
+            cols = ["data"]
         else:
             d1 = df1.as_pandas()
             d2 = df2.as_pandas()
         if not check_order:
-            d1 = d1.sort_values(df1.columns)
-            d2 = d2.sort_values(df1.columns)
+            d1 = d1.sort_values(cols)
+            d2 = d2.sort_values(cols)
         d1 = d1.reset_index(drop=True)
         d2 = d2.reset_index(drop=True)
         pd.testing.assert_frame_equal(
@@ -248,3 +251,68 @@ def get_join_schemas(
     else:
         aot(len(on) > 0, SchemaError("join on columns must be specified"))
     return cm, (df1.schema.union(schema2))
+
+
+def pa_table_as_array_iterable(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> Iterable[List[List[Any]]]:
+    """Convert a pyarrow table to an iterable of list
+
+    :param df: pyarrow table
+    :param columns: if not None, only these columns will be returned, defaults to None
+    :return: an iterable of list
+    """
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df.select(columns)
+    for batch in _df.to_batches():
+        for x in zip(*batch.to_pydict().values()):
+            yield list(x)
+
+
+def pa_table_as_array(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> List[List[List[Any]]]:
+    """Convert a pyarrow table to a list of list
+
+    :param df: pyarrow table
+    :param columns: if not None, only these columns will be returned, defaults to None
+    :return: a list of list
+    """
+    return list(pa_table_as_array_iterable(df, columns=columns))
+
+
+def pa_table_as_dict_iterable(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    """Convert a pyarrow table to an iterable of dict
+
+    :param df: pyarrow table
+    :param columns: if not None, only these columns will be returned, defaults to None
+    :return: an iterable of dict
+    """
+    for ck in _pa_table_as_dicts_chunks(df, columns=columns):
+        yield from ck
+
+
+def pa_table_as_dicts(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    """Convert a pyarrow table to a list of dict
+
+    :param df: pyarrow table
+    :param columns: if not None, only these columns will be returned, defaults to None
+    :return: a list of dict
+    """
+    res: List[Dict[str, Any]] = []
+    for ck in _pa_table_as_dicts_chunks(df, columns=columns):
+        res += ck
+    return res
+
+
+def _pa_table_as_dicts_chunks(
+    df: pa.Table, columns: Optional[List[str]] = None
+) -> Iterable[List[Dict[str, Any]]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df.select(columns)
+    for batch in _df.to_batches():
+        yield pa_batch_to_dicts(batch)

fugue/execution/execution_engine.py

@@ -1323,7 +1323,7 @@ class _Comap:
         self._on_init(partition_no, empty_dfs)
 
     def run(self, cursor: PartitionCursor, df: LocalDataFrame) -> LocalDataFrame:
-        data = list(df.as_dict_iterable())
+        data = df.as_dicts()
         if self.how == "inner":
             if len(data) < self.dfs_count:
                 return ArrayDataFrame([], self.output_schema)

fugue/execution/native_execution_engine.py

@@ -143,7 +143,7 @@ class PandasMapEngine(MapEngine):
             if (
                 isinstance(output_df, PandasDataFrame)
                 and output_df.schema != output_schema
-            ):
+            ):  # pragma: no cover
                 output_df = PandasDataFrame(output_df.native, output_schema)
             assert_or_throw(
                 output_df.schema == output_schema,

fugue/plugins.py

@@ -7,6 +7,7 @@ from fugue.dataframe import (
     as_array_iterable,
     as_arrow,
     as_dict_iterable,
+    as_dicts,
     as_pandas,
     drop_columns,
     fugue_annotated_param,

{fugue-0.8.7.dev4.dist-info → fugue-0.8.7.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.7.dev4
+Version: 0.8.7.dev6
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad ==0.9.2.dev3
+Requires-Dist: triad ==0.9.2.dev5
 Requires-Dist: adagio >=0.2.4
 Requires-Dist: qpd >=0.4.4
 Requires-Dist: fugue-sql-antlr >=0.1.6
@@ -32,7 +32,7 @@ Requires-Dist: fugue-sql-antlr[cpp] >=0.1.6 ; extra == 'all'
 Requires-Dist: pyspark >=3.1.1 ; extra == 'all'
 Requires-Dist: dask[dataframe,distributed] >=2023.5.0 ; extra == 'all'
 Requires-Dist: dask-sql ; extra == 'all'
-Requires-Dist: ray[data] >=2.1.0 ; extra == 'all'
+Requires-Dist: ray[data] >=2.4.0 ; extra == 'all'
 Requires-Dist: notebook ; extra == 'all'
 Requires-Dist: jupyterlab ; extra == 'all'
 Requires-Dist: ipython >=7.10.0 ; extra == 'all'
@@ -59,7 +59,7 @@ Requires-Dist: ipython >=7.10.0 ; extra == 'notebook'
 Provides-Extra: polars
 Requires-Dist: polars ; extra == 'polars'
 Provides-Extra: ray
-Requires-Dist: ray[data] >=2.1.0 ; extra == 'ray'
+Requires-Dist: ray[data] >=2.4.0 ; extra == 'ray'
 Requires-Dist: duckdb >=0.5.0 ; extra == 'ray'
 Requires-Dist: pyarrow >=6.0.1 ; extra == 'ray'
 Provides-Extra: spark
@@ -323,6 +323,7 @@ Feel free to message us on [Slack](http://slack.fugue.ai). We also have [contrib
 
 *   [How LyftLearn Democratizes Distributed Compute through Kubernetes Spark and Fugue](https://eng.lyft.com/how-lyftlearn-democratizes-distributed-compute-through-kubernetes-spark-and-fugue-c0875b97c3d9)
 *   [Clobotics - Large Scale Image Processing with Spark through Fugue](https://medium.com/fugue-project/large-scale-image-processing-with-spark-through-fugue-e510b9813da8)
+*   [Architecture for a data lake REST API using Delta Lake, Fugue & Spark (article by bitsofinfo)](https://bitsofinfo.wordpress.com/2023/08/14/data-lake-rest-api-delta-lake-fugue-spark)
 
 ### Mentioned Uses