fugue 0.8.2.dev1__py3-none-any.whl → 0.8.4__py3-none-any.whl

fugue/extensions/transformer/transformer.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Optional
 
 from fugue.dataframe import DataFrame, DataFrames, LocalDataFrame, ArrayDataFrame
 from fugue.extensions.context import ExtensionContext
@@ -47,6 +47,13 @@ class Transformer(ExtensionContext):
         """
         raise NotImplementedError
 
+    def get_format_hint(self) -> Optional[str]:
+        """Get the transformer's preferred data format, for example it can be
+        ``pandas``, ``pyarrow`` and None. This is to help the execution engine
+        use the most efficient way to execute the logic.
+        """
+        return None
+
     def on_init(self, df: DataFrame) -> None:  # pragma: no cover
         """Callback for initializing
         :ref:`physical partition that contains one or multiple logical partitions
@@ -147,6 +154,13 @@ class CoTransformer(ExtensionContext):
         """
         raise NotImplementedError
 
+    def get_format_hint(self) -> Optional[str]:  # pragma: no cover
+        """Get the transformer's preferred data format, for example it can be
+        ``pandas``, ``pyarrow`` and None. This is to help the execution engine
+        use the most efficient way to execute the logic.
+        """
+        return None
+
     def on_init(self, dfs: DataFrames) -> None:  # pragma: no cover
         """Callback for initializing
         :ref:`physical partition that contains one or multiple logical partitions

fugue/plugins.py

@@ -9,6 +9,7 @@ from fugue.dataframe import (
     as_dict_iterable,
     as_pandas,
     drop_columns,
+    fugue_annotated_param,
     get_column_names,
     get_schema,
     head,
@@ -29,6 +30,7 @@ from fugue.dataset import (
     is_empty,
     is_local,
 )
+from fugue.execution.api import as_fugue_engine_df
 from fugue.execution.factory import (
     infer_execution_engine,
     parse_execution_engine,

fugue/registry.py

@@ -1,19 +1,7 @@
-import inspect
-from typing import Any, Optional
-
-import pyarrow as pa
-
-from fugue._utils.interfaceless import (
-    DataFrameParam,
-    SimpleAnnotationConverter,
-    register_annotation_converter,
-)
-from fugue.dataframe import ArrowDataFrame, DataFrame
 from fugue.execution.factory import register_execution_engine, register_sql_engine
 from fugue.execution.native_execution_engine import (
     NativeExecutionEngine,
     QPDPandasEngine,
-    SqliteEngine,
 )
 
 
@@ -27,7 +15,6 @@ def _register() -> None:
         >>> import fugue
     """
     _register_engines()
-    _register_annotation_converters()
 
 
 def _register_engines() -> None:
@@ -37,35 +24,9 @@ def _register_engines() -> None:
     register_execution_engine(
         "pandas", lambda conf: NativeExecutionEngine(conf), on_dup="ignore"
     )
-    register_sql_engine("sqlite", lambda engine: SqliteEngine(engine), on_dup="ignore")
     register_sql_engine(
         "qpdpandas", lambda engine: QPDPandasEngine(engine), on_dup="ignore"
     )
     register_sql_engine(
         "qpd_pandas", lambda engine: QPDPandasEngine(engine), on_dup="ignore"
     )
-
-
-def _register_annotation_converters() -> None:
-    register_annotation_converter(
-        0.8,
-        SimpleAnnotationConverter(
-            pa.Table,
-            lambda param: _PyArrowTableParam(param),
-        ),
-    )
-
-
-class _PyArrowTableParam(DataFrameParam):
-    def __init__(self, param: Optional[inspect.Parameter]):
-        super().__init__(param, annotation="Table")
-
-    def to_input_data(self, df: DataFrame, ctx: Any) -> Any:
-        return df.as_arrow()
-
-    def to_output_df(self, output: Any, schema: Any, ctx: Any) -> DataFrame:
-        assert isinstance(output, pa.Table)
-        return ArrowDataFrame(output, schema=schema)
-
-    def count(self, df: Any) -> int:  # pragma: no cover
-        return df.count()

fugue/sql/_utils.py

@@ -5,7 +5,7 @@ import jinja2
 from jinja2 import Template
 from triad import assert_or_throw
 
-from ..collections import Yielded
+from ..collections.yielded import Yielded
 from ..exceptions import FugueSQLError
 from ..workflow.workflow import FugueWorkflow, WorkflowDataFrame
 

fugue/workflow/_checkpoint.py

@@ -166,7 +166,7 @@ class CheckpointPath(object):
 
     def get_table_name(self, obj_id: str, permanent: bool) -> str:
         path = self._path if permanent else self._temp_path
-        return to_uuid(path, obj_id)[:5]
+        return "temp_" + to_uuid(path, obj_id)[:5]
 
     def temp_file_exists(self, path: str) -> bool:
         try:

fugue/workflow/api.py

@@ -49,11 +49,11 @@ def transform(  # noqa: C901
 ) -> Any:
     """Transform this dataframe using transformer. It's a wrapper of
     :meth:`~fugue.workflow.workflow.FugueWorkflow.transform` and
-    :meth:`~fugue.workflow.workflow.FugueWorkflow.run`. It let you do the
-    basic dataframe transformation without using
+    :meth:`~fugue.workflow.workflow.FugueWorkflow.run`. It will let you do
+    the basic dataframe transformation without using
     :class:`~fugue.workflow.workflow.FugueWorkflow` and
-    :class:`~fugue.dataframe.dataframe.DataFrame`. Both input and output
-    can be native types only.
+    :class:`~fugue.dataframe.dataframe.DataFrame`. Also, only native
+    types are accepted for both input and output.
 
     Please read |TransformerTutorial|
 
@@ -80,8 +80,8 @@ def transform(  # noqa: C901
     :param engine_conf: |ParamsLikeObject|, defaults to None
     :param as_fugue: If true, the function will always return
         a ``FugueDataFrame``, otherwise, if ``df`` is in native dataframe types such
-        as pandas dataframe, then the output will also in its native format. Defaults
-        to False
+        as pandas dataframe, then the output will also return in its native format.
+        Defaults to False
     :param persist: Whether to persist(materialize) the dataframe before returning
     :param as_local: If true, the result will be converted to a ``LocalDataFrame``
     :param save_path: Whether to save the output to a file (see the note)
@@ -109,7 +109,7 @@ def transform(  # noqa: C901
 
         * When `save_path` is None and `checkpoint` is False, then the output will
           not be saved into a file. The return will be a dataframe.
-        * When `save_path` is None and `checkpoint` is True, then the output will be
+        * When `save_path` is None and `checkpoint` is True, then the output is
           saved into the path set by `fugue.workflow.checkpoint.path`, the name will
           be randomly chosen, and it is NOT a deterministic checkpoint, so if you run
           multiple times, the output will be saved into different files. The return
@@ -196,21 +196,21 @@ def out_transform(
 ) -> None:
     """Transform this dataframe using transformer. It's a wrapper of
     :meth:`~fugue.workflow.workflow.FugueWorkflow.out_transform` and
-    :meth:`~fugue.workflow.workflow.FugueWorkflow.run`. It let you do the
+    :meth:`~fugue.workflow.workflow.FugueWorkflow.run`. It will let you do the
     basic dataframe transformation without using
     :class:`~fugue.workflow.workflow.FugueWorkflow` and
-    :class:`~fugue.dataframe.dataframe.DataFrame`. The input can be native
-    type only
+    :class:`~fugue.dataframe.dataframe.DataFrame`. Only native types are
+    accepted for both input and output.
 
     Please read |TransformerTutorial|
 
     :param df: |DataFrameLikeObject| or :class:`~fugue.workflow.yielded.Yielded`
         or a path string to a parquet file
     :param using: transformer-like object, can't be a string expression
-    :param params: |ParamsLikeObject| to run the processor, defaults to None.
+    :param params: |ParamsLikeObject| to run the processor, defaults to None
         The transformer will be able to access this value from
         :meth:`~fugue.extensions.context.ExtensionContext.params`
-    :param partition: |PartitionLikeObject|, defaults to None.
+    :param partition: |PartitionLikeObject|, defaults to None
     :param callback: |RPCHandlerLikeObject|, defaults to None
     :param ignore_errors: list of exception types the transformer can ignore,
         defaults to None (empty list)
@@ -225,7 +225,7 @@ def out_transform(
 
     .. note::
 
-        This function can only take parquet file paths in `df`. Csv and other file
+        This function can only take parquet file paths in `df`. CSV and JSON file
         formats are disallowed.
 
         This transformation is guaranteed to execute immediately (eager)

fugue/workflow/module.py

@@ -2,12 +2,18 @@ import copy
 import inspect
 from typing import Any, Callable, Dict, Iterable, Optional
 
-from fugue._utils.interfaceless import FunctionWrapper, _FuncParam
-from fugue.exceptions import FugueInterfacelessError
-from fugue.workflow.workflow import FugueWorkflow, WorkflowDataFrame, WorkflowDataFrames
+from triad import extension_method
+from triad.collections.function_wrapper import (
+    AnnotatedParam,
+    FunctionWrapper,
+    function_wrapper,
+)
 from triad.utils.assertion import assert_or_throw
 from triad.utils.convert import get_caller_global_local_vars, to_function
-from triad import extension_method
+
+from fugue.constants import FUGUE_ENTRYPOINT
+from fugue.exceptions import FugueInterfacelessError
+from fugue.workflow.workflow import FugueWorkflow, WorkflowDataFrame, WorkflowDataFrames
 
 
 def module(
@@ -50,23 +56,9 @@ def _to_module(
     raise FugueInterfacelessError(f"{obj} is not a valid module", exp)
 
 
-class _FugueWorkflowParam(_FuncParam):
-    def __init__(self, param: Optional[inspect.Parameter]):
-        super().__init__(param, "FugueWorkflow", "w")
-
-
-class _WorkflowDataFrameParam(_FuncParam):
-    def __init__(self, param: Optional[inspect.Parameter]):
-        super().__init__(param, "WorkflowDataFrame", "v")
-
-
-class _WorkflowDataFramesParam(_FuncParam):
-    def __init__(self, param: Optional[inspect.Parameter]):
-        super().__init__(param, "WorkflowDataFrame", "u")
-
-
+@function_wrapper(FUGUE_ENTRYPOINT)
 class _ModuleFunctionWrapper(FunctionWrapper):
-    def __init__(
+    def __init__(  # pylint: disable-all
         self,
         func: Callable,
         params_re: str = "^(w?(u|v+)|w(u?|v*))x*z?$",
@@ -153,20 +145,21 @@ class _ModuleFunctionWrapper(FunctionWrapper):
                     wf = v.workflow
         return wf
 
-    def _parse_param(
-        self,
-        annotation: Any,
-        param: Optional[inspect.Parameter],
-        none_as_other: bool = True,
-    ) -> _FuncParam:
-        if issubclass(annotation, FugueWorkflow):
-            # to prevent cyclic import
-            return _FugueWorkflowParam(param)
-        elif annotation == WorkflowDataFrame:
-            # to prevent cyclic import
-            return _WorkflowDataFrameParam(param)
-        elif annotation == WorkflowDataFrames:
-            # to prevent cyclic import
-            return _WorkflowDataFramesParam(param)
-        else:
-            return super()._parse_param(annotation, param, none_as_other)
+
+@_ModuleFunctionWrapper.annotated_param(
+    FugueWorkflow,
+    "w",
+    matcher=lambda x: inspect.isclass(x) and issubclass(x, FugueWorkflow),
+)
+class _FugueWorkflowParam(AnnotatedParam):
+    pass
+
+
+@_ModuleFunctionWrapper.annotated_param(WorkflowDataFrame, "v")
+class _WorkflowDataFrameParam(AnnotatedParam):
+    pass
+
+
+@_ModuleFunctionWrapper.annotated_param(WorkflowDataFrames, "u")
+class _WorkflowDataFramesParam(AnnotatedParam):
+    pass

fugue/workflow/workflow.py

@@ -1348,6 +1348,12 @@ class WorkflowDataFrame(DataFrame):
         """
         raise NotImplementedError("WorkflowDataFrame does not support this method")
 
+    def as_local_bounded(self) -> DataFrame:  # type: ignore  # pragma: no cover
+        """
+        :raises NotImplementedError: don't call this method
+        """
+        raise NotImplementedError("WorkflowDataFrame does not support this method")
+
     @property
     def is_bounded(self) -> bool:  # pragma: no cover
         """

{fugue-0.8.2.dev1.dist-info → fugue-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.2.dev1
+Version: 0.8.4
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -20,17 +20,16 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
-Requires-Dist: triad (>=0.8.1)
+Requires-Dist: triad (>=0.8.8)
 Requires-Dist: adagio (>=0.2.4)
-Requires-Dist: qpd (>=0.4.0)
-Requires-Dist: fugue-sql-antlr (>=0.1.5)
-Requires-Dist: sqlalchemy
-Requires-Dist: sqlglot
+Requires-Dist: qpd (>=0.4.1)
+Requires-Dist: fugue-sql-antlr (>=0.1.6)
 Requires-Dist: pyarrow (>=0.15.1)
-Requires-Dist: pandas (>=1.0.2)
+Requires-Dist: pandas (>=1.2.0)
+Requires-Dist: sqlglot
 Requires-Dist: jinja2
 Provides-Extra: all
-Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.5) ; extra == 'all'
+Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.6) ; extra == 'all'
 Requires-Dist: pyspark ; extra == 'all'
 Requires-Dist: ray[data] (>=2.0.0) ; extra == 'all'
 Requires-Dist: qpd[dask] (>=0.4.0) ; extra == 'all'
@@ -39,14 +38,15 @@ Requires-Dist: jupyterlab ; extra == 'all'
 Requires-Dist: ipython (>=7.10.0) ; extra == 'all'
 Requires-Dist: duckdb (>=0.5.0) ; extra == 'all'
 Requires-Dist: pyarrow (>=6.0.1) ; extra == 'all'
+Requires-Dist: polars ; extra == 'all'
 Requires-Dist: dask[dataframe,distributed] ; (python_version < "3.8") and extra == 'all'
 Requires-Dist: ibis-framework (>=2.1.1) ; (python_version < "3.8") and extra == 'all'
 Requires-Dist: dask[dataframe,distributed] (>=2022.9.0) ; (python_version >= "3.8") and extra == 'all'
 Requires-Dist: ibis-framework (>=3.2.0) ; (python_version >= "3.8") and extra == 'all'
 Provides-Extra: cpp_sql_parser
-Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.5) ; extra == 'cpp_sql_parser'
+Requires-Dist: fugue-sql-antlr[cpp] (>=0.1.6) ; extra == 'cpp_sql_parser'
 Provides-Extra: dask
-Requires-Dist: qpd[dask] (>=0.4.0) ; extra == 'dask'
+Requires-Dist: qpd[dask] (>=0.4.1) ; extra == 'dask'
 Requires-Dist: dask[dataframe,distributed] ; (python_version < "3.8") and extra == 'dask'
 Requires-Dist: dask[dataframe,distributed] (>=2022.9.0) ; (python_version >= "3.8") and extra == 'dask'
 Provides-Extra: duckdb
@@ -60,6 +60,8 @@ Provides-Extra: notebook
 Requires-Dist: notebook ; extra == 'notebook'
 Requires-Dist: jupyterlab ; extra == 'notebook'
 Requires-Dist: ipython (>=7.10.0) ; extra == 'notebook'
+Provides-Extra: polars
+Requires-Dist: polars ; extra == 'polars'
 Provides-Extra: ray
 Requires-Dist: ray[data] (>=2.0.0) ; extra == 'ray'
 Requires-Dist: duckdb (>=0.5.0) ; extra == 'ray'
@@ -76,9 +78,9 @@ Requires-Dist: pyspark ; extra == 'spark'
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/4fa5f2f53e6f48aaa1218a89f4808b91)](https://www.codacy.com/gh/fugue-project/fugue/dashboard?utm_source=github.com&utm_medium=referral&utm_content=fugue-project/fugue&utm_campaign=Badge_Grade)
 [![Downloads](https://pepy.tech/badge/fugue)](https://pepy.tech/project/fugue)
 
-| Tutorials | API Documentation | Chat with us on slack! |
-| --- | --- | --- |
-| [![Jupyter Book Badge](https://jupyterbook.org/badge.svg)](https://fugue-tutorials.readthedocs.io/) | [![Doc](https://readthedocs.org/projects/fugue/badge)](https://fugue.readthedocs.org)  | [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](http://slack.fugue.ai) |
+| Tutorials                                                                                           | API Documentation                                                                     | Chat with us on slack!                                                                                                   |
+| --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
+| [![Jupyter Book Badge](https://jupyterbook.org/badge.svg)](https://fugue-tutorials.readthedocs.io/) | [![Doc](https://readthedocs.org/projects/fugue/badge)](https://fugue.readthedocs.org) | [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](http://slack.fugue.ai) |
 
 
 **Fugue is a unified interface for distributed computing that lets users execute Python, Pandas, and SQL code on Spark, Dask, and Ray with minimal rewrites**.
@@ -217,13 +219,14 @@ It also has the following installation extras:
 *   **dask**: to support Dask as the ExecutionEngine.
 *   **ray**: to support Ray as the ExecutionEngine.
 *   **duckdb**: to support DuckDB as the ExecutionEngine, read [details](https://fugue-tutorials.readthedocs.io/tutorials/integrations/backends/duckdb.html).
+*   **polars**: to support Polars DataFrames and extensions using Polars.
 *   **ibis**: to enable Ibis for Fugue workflows, read [details](https://fugue-tutorials.readthedocs.io/tutorials/integrations/backends/ibis.html).
 *   **cpp_sql_parser**: to enable the CPP antlr parser for Fugue SQL. It can be 50+ times faster than the pure Python parser. For the main Python versions and platforms, there is already pre-built binaries, but for the remaining, it needs a C++ compiler to build on the fly.
 
 For example a common use case is:
 
 ```bash
-pip install fugue[duckdb,spark]
+pip install "fugue[duckdb,spark]"
 ```
 
 Note if you already installed Spark or DuckDB independently, Fugue is able to automatically use them without installing the extras.
@@ -270,6 +273,7 @@ By being an abstraction layer, Fugue can be used with a lot of other open-source
 Python backends:
 
 *   [Pandas](https://github.com/pandas-dev/pandas)
+*   [Polars](https://www.pola.rs) (DataFrames only)
 *   [Spark](https://github.com/apache/spark)
 *   [Dask](https://github.com/dask/dask)
 *   [Ray](http://github.com/ray-project/ray)
@@ -281,6 +285,7 @@ FugueSQL backends:
 *   [Duckdb](https://github.com/duckdb/duckdb) - in-process SQL OLAP database management
 *   [dask-sql](https://github.com/dask-contrib/dask-sql) - SQL interface for Dask
 *   SparkSQL
+*   BigQuery
 
 
 Fugue is available as a backend or can integrate with the following projects:
@@ -291,23 +296,36 @@ Fugue is available as a backend or can integrate with the following projects:
 *   [Prefect](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/prefect.html) - workflow orchestration
 *   [Pandera](https://fugue-tutorials.readthedocs.io/tutorials/integrations/ecosystem/pandera.html) - data validation
 
+Registered 3rd party extensions (majorly for Fugue SQL) include:
 
-## Further Resources
+*   [Pandas plot](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.plot.html) - visualize data using matplotlib or plotly
+*   [Seaborn](https://seaborn.pydata.org/api.html) - visualize data using seaborn
+*   [WhyLogs](https://whylogs.readthedocs.io/en/latest/examples/integrations/Fugue_Profiling.html?highlight=fugue) - visualize data profiling
+*   [Vizzu](https://github.com/vizzuhq/ipyvizzu) - visualize data using ipyvizzu
 
-View some of our latest conferences presentations and content. For a more complete list, check the [Content](https://fugue-tutorials.readthedocs.io/tutorials/resources/content.html) page in the tutorials.
+## Community and Contributing
+
+Feel free to message us on [Slack](http://slack.fugue.ai). We also have [contributing instructions](CONTRIBUTING.md).
 
 ### Case Studies
 
 *   [How LyftLearn Democratizes Distributed Compute through Kubernetes Spark and Fugue](https://eng.lyft.com/how-lyftlearn-democratizes-distributed-compute-through-kubernetes-spark-and-fugue-c0875b97c3d9)
 *   [Clobotics - Large Scale Image Processing with Spark through Fugue](https://medium.com/fugue-project/large-scale-image-processing-with-spark-through-fugue-e510b9813da8)
 
+### Mentioned Uses
+
+*   [Productionizing Data Science at Interos, Inc. (LinkedIn post by Anthony Holten)](https://www.linkedin.com/posts/anthony-holten_pandas-spark-dask-activity-7022628193983459328-QvcF)
+
+*   [Multiple Time Series Forecasting with Fugue & Nixtla at Bain & Company(LinkedIn post by Fahad Akbar)](https://www.linkedin.com/posts/fahadakbar_fugue-datascience-forecasting-activity-7041119034813124608-u08q?utm_source=share&utm_medium=member_desktop)
+
+## Further Resources
+
+View some of our latest conferences presentations and content. For a more complete list, check the [Content](https://fugue-tutorials.readthedocs.io/tutorials/resources/content.html) page in the tutorials.
+
 ### Blogs
 
 *   [Why Pandas-like Interfaces are Sub-optimal for Distributed Computing](https://towardsdatascience.com/why-pandas-like-interfaces-are-sub-optimal-for-distributed-computing-322dacbce43)
-*   [Interoperable Python and SQL in Jupyter Notebooks (Towards Data Science)](https://towardsdatascience.com/interoperable-python-and-sql-in-jupyter-notebooks-86245e711352)
-*   [Introducing Fugue - Reducing PySpark Developer Friction](https://towardsdatascience.com/introducing-fugue-reducing-pyspark-developer-friction-a702230455de)
 *   [Introducing FugueSQL — SQL for Pandas, Spark, and Dask DataFrames (Towards Data Science by Khuyen Tran)](https://towardsdatascience.com/introducing-fuguesql-sql-for-pandas-spark-and-dask-dataframes-63d461a16b27)
-*   [Using Pandera on Spark for Data Validation through Fugue (Towards Data Science)](https://towardsdatascience.com/using-pandera-on-spark-for-data-validation-through-fugue-72956f274793)
 
 ### Conferences
 
@@ -317,7 +335,3 @@ View some of our latest conferences presentations and content. For a more comple
 *   [FugueSQL - The Enhanced SQL Interface for Pandas, Spark, and Dask DataFrames (PyData Global)](https://www.youtube.com/watch?v=OBpnGYjNBBI)
 *   [Distributed Hybrid Parameter Tuning](https://www.youtube.com/watch?v=_GBjqskD8Qk)
 
-## Community and Contributing
-
-Feel free to message us on [Slack](http://slack.fugue.ai). We also have [contributing instructions](CONTRIBUTING.md).
-