maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/dataframe/operators.py

@@ -0,0 +1,273 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ..core import ENTITY_TYPE, OutputType
+from ..core.operator import (
+    Fuse,
+    FuseChunkMixin,
+    Operator,
+    ShuffleProxy,
+    TileableOperatorMixin,
+)
+from ..tensor.core import TENSOR_TYPE
+from ..tensor.datasource import tensor as astensor
+from .core import DATAFRAME_TYPE, SERIES_TYPE
+
+
+class DataFrameOperatorMixin(TileableOperatorMixin):
+    __slots__ = ()
+    _op_module_ = "dataframe"
+
+    def new_dataframes(
+        self,
+        inputs,
+        shape=None,
+        dtypes=None,
+        index_value=None,
+        columns_value=None,
+        chunks=None,
+        nsplits=None,
+        output_limit=None,
+        kws=None,
+        **kw
+    ):
+        setattr(self, "_output_types", [OutputType.dataframe])
+        return self.new_tileables(
+            inputs,
+            shape=shape,
+            dtypes=dtypes,
+            index_value=index_value,
+            columns_value=columns_value,
+            chunks=chunks,
+            nsplits=nsplits,
+            output_limit=output_limit,
+            kws=kws,
+            **kw
+        )
+
+    def new_dataframe(
+        self,
+        inputs,
+        shape=None,
+        dtypes=None,
+        index_value=None,
+        columns_value=None,
+        **kw
+    ):
+        if getattr(self, "output_limit") != 1:
+            raise TypeError("cannot new DataFrame with more than 1 outputs")
+
+        return self.new_dataframes(
+            inputs,
+            shape=shape,
+            dtypes=dtypes,
+            index_value=index_value,
+            columns_value=columns_value,
+            **kw
+        )[0]
+
+    def new_seriess(
+        self,
+        inputs,
+        shape=None,
+        dtype=None,
+        index_value=None,
+        name=None,
+        chunks=None,
+        nsplits=None,
+        output_limit=None,
+        kws=None,
+        **kw
+    ):
+        setattr(self, "_output_types", [OutputType.series])
+        return self.new_tileables(
+            inputs,
+            shape=shape,
+            dtype=dtype,
+            index_value=index_value,
+            name=name,
+            chunks=chunks,
+            nsplits=nsplits,
+            output_limit=output_limit,
+            kws=kws,
+            **kw
+        )
+
+    def new_series(
+        self, inputs, shape=None, dtype=None, index_value=None, name=None, **kw
+    ):
+        if getattr(self, "output_limit") != 1:
+            raise TypeError("cannot new Series with more than 1 outputs")
+
+        return self.new_seriess(
+            inputs, shape=shape, dtype=dtype, index_value=index_value, name=name, **kw
+        )[0]
+
+    def new_df_or_series(self, inputs, **kw):
+        setattr(self, "_output_types", [OutputType.df_or_series])
+        return self.new_tileables(inputs, **kw)[0]
+
+    def new_indexes(
+        self,
+        inputs,
+        shape=None,
+        dtype=None,
+        index_value=None,
+        name=None,
+        chunks=None,
+        nsplits=None,
+        output_limit=None,
+        kws=None,
+        **kw
+    ):
+        setattr(self, "_output_types", [OutputType.index])
+        return self.new_tileables(
+            inputs,
+            shape=shape,
+            dtype=dtype,
+            index_value=index_value,
+            name=name,
+            chunks=chunks,
+            nsplits=nsplits,
+            output_limit=output_limit,
+            kws=kws,
+            **kw
+        )
+
+    def new_index(
+        self, inputs, shape=None, dtype=None, index_value=None, name=None, **kw
+    ):
+        if getattr(self, "output_limit") != 1:
+            raise TypeError("cannot new Index with more than 1 outputs")
+
+        return self.new_indexes(
+            inputs, shape=shape, dtype=dtype, index_value=index_value, name=name, **kw
+        )[0]
+
+    def new_scalars(
+        self, inputs, dtype=None, chunks=None, output_limit=None, kws=None, **kw
+    ):
+        setattr(self, "_output_types", [OutputType.scalar])
+        return self.new_tileables(
+            inputs,
+            shape=(),
+            dtype=dtype,
+            chunks=chunks,
+            nsplits=(),
+            output_limit=output_limit,
+            kws=kws,
+            **kw
+        )
+
+    def new_scalar(self, inputs, dtype=None, **kw):
+        if getattr(self, "output_limit") != 1:
+            raise TypeError("cannot new tensor with more than 1 outputs")
+
+        return self.new_scalars(inputs, dtype=dtype, **kw)[0]
+
+    def new_categoricals(
+        self,
+        inputs,
+        shape=None,
+        dtype=None,
+        categories_value=None,
+        chunks=None,
+        nsplits=None,
+        output_limit=None,
+        kws=None,
+        **kw
+    ):
+        setattr(self, "_output_types", [OutputType.categorical])
+        return self.new_tileables(
+            inputs,
+            shape=shape,
+            dtype=dtype,
+            categories_value=categories_value,
+            chunks=chunks,
+            nsplits=nsplits,
+            output_limit=output_limit,
+            kws=kws,
+            **kw
+        )
+
+    def new_categorical(
+        self, inputs, shape=None, dtype=None, categories_value=None, **kw
+    ):
+        if getattr(self, "output_limit") != 1:
+            raise TypeError("cannot new Categorical with more than 1 outputs")
+
+        return self.new_categoricals(
+            inputs, shape=shape, dtype=dtype, categories_value=categories_value, **kw
+        )[0]
+
+    @classmethod
+    def _process_groupby_params(cls, groupby_params):
+        new_groupby_params = groupby_params.copy()
+        if isinstance(groupby_params["by"], list):
+            by = []
+            for v in groupby_params["by"]:
+                if isinstance(v, ENTITY_TYPE):
+                    by.append(cls.concat_tileable_chunks(v).chunks[0])
+                else:
+                    by.append(v)
+            new_groupby_params["by"] = by
+        return new_groupby_params
+
+    @classmethod
+    def _get_groupby_inputs(cls, groupby, groupby_params):
+        inputs = [groupby]
+        chunk_inputs = list(groupby.chunks)
+        if isinstance(groupby_params["by"], list):
+            for chunk_v, v in zip(
+                groupby_params["by"], groupby.op.groupby_params["by"]
+            ):
+                if isinstance(v, ENTITY_TYPE):
+                    inputs.append(v)
+                    chunk_inputs.append(chunk_v)
+        return inputs, chunk_inputs
+
+    @staticmethod
+    def _process_input(x):
+        from .initializer import DataFrame, Series
+
+        if isinstance(x, (DATAFRAME_TYPE, SERIES_TYPE)) or pd.api.types.is_scalar(x):
+            return x
+        elif isinstance(x, pd.Series):
+            return Series(x)
+        elif isinstance(x, pd.DataFrame):
+            return DataFrame(x)
+        elif isinstance(x, (list, tuple, np.ndarray, TENSOR_TYPE)):
+            return astensor(x)
+        raise NotImplementedError
+
+
+DataFrameOperator = Operator
+
+
+class DataFrameShuffleProxy(ShuffleProxy, DataFrameOperatorMixin):
+    def __init__(self, sparse=None, output_types=None, **kwargs):
+        super().__init__(sparse=sparse, _output_types=output_types, **kwargs)
+
+
+class DataFrameFuseChunkMixin(FuseChunkMixin, DataFrameOperatorMixin):
+    __slots__ = ()
+
+
+class DataFrameFuseChunk(Fuse, DataFrameFuseChunkMixin):
+    @property
+    def output_types(self):
+        return self.outputs[-1].chunk.op.output_types

maxframe/dataframe/plotting/__init__.py

@@ -0,0 +1,40 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def _install():
+    import pandas as pd
+
+    from ...utils import adapt_docstring
+    from ..core import DATAFRAME_TYPE, SERIES_TYPE
+    from ..misc.accessor import CachedAccessor
+    from .core import DataFramePlotAccessor, PlotAccessor, SeriesPlotAccessor
+
+    for t in DATAFRAME_TYPE:
+        t.plot = CachedAccessor("plot", DataFramePlotAccessor)
+    for method in dir(pd.DataFrame.plot):
+        if not method.startswith("_"):
+            DataFramePlotAccessor._register(method)
+
+    for t in SERIES_TYPE:
+        t.plot = CachedAccessor("plot", SeriesPlotAccessor)
+    for method in dir(pd.Series.plot):
+        if not method.startswith("_"):
+            SeriesPlotAccessor._register(method)
+
+    PlotAccessor.__doc__ = adapt_docstring(pd.DataFrame.plot.__doc__)
+
+
+_install()
+del _install

maxframe/dataframe/plotting/core.py

@@ -0,0 +1,78 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+
+import pandas as pd
+
+from ...core import ENTITY_TYPE, ExecutableTuple
+from ...utils import adapt_docstring, get_item_if_scalar
+
+
+class PlotAccessor:
+    def __init__(self, obj):
+        self._obj = obj
+
+    def __call__(self, kind="line", session=None, **kwargs):
+        to_executes = self._get_executables(**kwargs)
+
+        result = dict()
+        executed = (
+            ExecutableTuple(to_executes.values())
+            .execute(session=session)
+            .fetch(session=session)
+        )
+        for p, v in zip(to_executes, executed):
+            result[p] = get_item_if_scalar(v)
+
+        data = result.pop("__object__")
+        pd_kwargs = kwargs.copy()
+        pd_kwargs["kind"] = kind
+        pd_kwargs.update(result)
+
+        return data.plot(**pd_kwargs)
+
+    def _get_executables(self, **kwargs) -> OrderedDict:
+        to_executes = OrderedDict()
+        to_executes["__object__"] = self._obj
+
+        for k, v in kwargs.items():
+            if isinstance(v, ENTITY_TYPE):
+                to_executes[k] = v
+
+        return to_executes
+
+    @classmethod
+    def _gen_func(cls, name, doc):
+        def _inner(self, *args, **kwargs):
+            return self(kind=name, *args, **kwargs)
+
+        _inner.__name__ = name
+        _inner.__doc__ = doc
+
+        return _inner
+
+    @classmethod
+    def _register(cls, method):
+        doc = getattr(pd.DataFrame.plot, method).__doc__
+        new_doc = adapt_docstring(doc)
+        setattr(cls, method, cls._gen_func(method, new_doc))
+
+
+class DataFramePlotAccessor(PlotAccessor):
+    pass
+
+
+class SeriesPlotAccessor(PlotAccessor):
+    pass

maxframe/dataframe/plotting/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/plotting/tests/test_plotting.py

@@ -0,0 +1,136 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tempfile
+import warnings
+
+import mock
+import numpy as np
+import pandas as pd
+import pytest
+
+try:
+    import matplotlib
+except ImportError:  # pragma: no cover
+    matplotlib = None
+
+from .... import dataframe as md
+from .... import tensor as mt
+from ....core import ExecutableTuple
+from ....tensor.datasource import Scalar
+from ...datasource import PandasDataSourceOperator
+
+
+@pytest.fixture
+def wrap_execute():
+    exec_container = dict()
+
+    def new_execute(executable_tuple: ExecutableTuple, *_args, **_kw):
+        for entity in executable_tuple:
+            if isinstance(entity.op, PandasDataSourceOperator):
+                exec_container[entity] = entity.op.get_data()
+            elif isinstance(entity.op, Scalar):
+                exec_container[entity] = entity.op.data
+            else:
+                raise NotImplementedError(entity.op)
+        return executable_tuple
+
+    def new_fetch(executable_tuple: ExecutableTuple, *_args, **_kw):
+        return tuple(exec_container[et] for et in executable_tuple)
+
+    with mock.patch(
+        "maxframe.core.entity.executable.ExecutableTuple.execute", new=new_execute
+    ), mock.patch(
+        "maxframe.core.entity.executable.ExecutableTuple.fetch", new=new_fetch
+    ):
+        yield
+
+
+def close(fignum=None):  # pragma: no cover
+    from matplotlib.pyplot import close as _close
+    from matplotlib.pyplot import get_fignums
+
+    if fignum is None:
+        for fignum in get_fignums():
+            _close(fignum)
+    else:
+        _close(fignum)
+
+
+def assert_is_valid_plot_return_object(objs):  # pragma: no cover
+    import matplotlib.pyplot as plt
+
+    if isinstance(objs, (pd.Series, np.ndarray)):
+        for el in objs.ravel():
+            msg = (
+                "one of 'objs' is not a matplotlib Axes instance, "
+                f"type encountered {type(el).__name__}"
+            )
+            assert isinstance(el, (plt.Axes, dict)), msg
+    else:
+        msg = (
+            "objs is neither an ndarray of Artist instances nor a single "
+            f"ArtistArtist instance, tuple, or dict, 'objs' is a {type(objs).__name__}"
+        )
+        assert isinstance(objs, (plt.Artist, tuple, dict)), msg
+
+
+def _check_plot_works(f, filterwarnings="always", **kwargs):  # pragma: no cover
+    import matplotlib.pyplot as plt
+
+    ret = None
+    with warnings.catch_warnings():
+        warnings.simplefilter(filterwarnings)
+        try:
+            try:
+                fig = kwargs["figure"]
+            except KeyError:
+                fig = plt.gcf()
+
+            plt.clf()
+
+            kwargs.get("ax", fig.add_subplot(211))
+            ret = f(**kwargs)
+
+            assert_is_valid_plot_return_object(ret)
+
+            if f is pd.plotting.bootstrap_plot:
+                assert "ax" not in kwargs
+            else:
+                kwargs["ax"] = fig.add_subplot(212)
+
+            ret = f(**kwargs)
+            assert_is_valid_plot_return_object(ret)
+
+            with tempfile.TemporaryFile() as path:
+                plt.savefig(path)
+        finally:
+            close(fig)
+
+        return ret
+
+
+@pytest.mark.skipif(matplotlib is None, reason="matplotlib is not installed")
+def test_plotting(wrap_execute):
+    raw = pd.DataFrame(
+        {
+            "a": ["s" + str(i) for i in range(10)],
+            "b": np.random.RandomState(0).randint(10, size=10),
+        }
+    )
+    df = md.DataFrame(raw)
+
+    _check_plot_works(df.plot, x="a", y="b")
+    _check_plot_works(df.plot, x="a", y=mt.tensor("b"))
+    _check_plot_works(df.plot.line)

maxframe/dataframe/reduction/__init__.py

@@ -0,0 +1,107 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .aggregation import DataFrameAggregate
+from .all import DataFrameAll
+from .any import DataFrameAny
+from .core import CustomReduction
+from .count import DataFrameCount
+from .cummax import DataFrameCummax
+from .cummin import DataFrameCummin
+from .cumprod import DataFrameCumprod
+from .cumsum import DataFrameCumsum
+from .custom_reduction import DataFrameCustomReduction
+from .kurtosis import DataFrameKurtosis
+from .max import DataFrameMax
+from .mean import DataFrameMean
+from .min import DataFrameMin
+from .nunique import DataFrameNunique
+from .prod import DataFrameProd
+from .reduction_size import DataFrameSize
+from .sem import DataFrameSem
+from .skew import DataFrameSkew
+from .str_concat import DataFrameStrConcat, build_str_concat_object
+from .sum import DataFrameSum
+from .unique import DataFrameUnique, unique
+from .var import DataFrameVar
+
+
+def _install():
+    from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
+    from .aggregation import aggregate
+    from .all import all_dataframe, all_index, all_series
+    from .any import any_dataframe, any_index, any_series
+    from .count import count_dataframe, count_series
+    from .cummax import cummax
+    from .cummin import cummin
+    from .cumprod import cumprod
+    from .cumsum import cumsum
+    from .kurtosis import kurt_dataframe, kurt_series
+    from .max import max_dataframe, max_index, max_series
+    from .mean import mean_dataframe, mean_series
+    from .min import min_dataframe, min_index, min_series
+    from .nunique import nunique_dataframe, nunique_series
+    from .prod import prod_dataframe, prod_series
+    from .reduction_size import size_dataframe, size_series
+    from .sem import sem_dataframe, sem_series
+    from .skew import skew_dataframe, skew_series
+    from .std import std_dataframe, std_series
+    from .sum import sum_dataframe, sum_series
+    from .var import var_dataframe, var_series
+
+    funcs = [
+        ("sum", sum_series, sum_dataframe),
+        ("prod", prod_series, prod_dataframe),
+        ("product", prod_series, prod_dataframe),
+        ("max", max_series, max_dataframe),
+        ("min", min_series, min_dataframe),
+        ("count", count_series, count_dataframe),
+        ("mean", mean_series, mean_dataframe),
+        ("var", var_series, var_dataframe),
+        ("std", std_series, std_dataframe),
+        ("all", all_series, all_dataframe),
+        ("any", any_series, any_dataframe),
+        ("cummax", cummax, cummax),
+        ("cummin", cummin, cummin),
+        ("cumprod", cumprod, cumprod),
+        ("cumsum", cumsum, cumsum),
+        ("agg", aggregate, aggregate),
+        ("aggregate", aggregate, aggregate),
+        ("nunique", nunique_series, nunique_dataframe),
+        ("sem", sem_series, sem_dataframe),
+        ("skew", skew_series, skew_dataframe),
+        ("kurt", kurt_series, kurt_dataframe),
+        ("kurtosis", kurt_series, kurt_dataframe),
+        ("unique", unique, None),
+        ("_reduction_size", size_dataframe, size_series),
+    ]
+    for func_name, series_func, df_func in funcs:
+        if df_func is not None:  # pragma: no branch
+            for t in DATAFRAME_TYPE:
+                setattr(t, func_name, df_func)
+        if series_func is not None:  # pragma: no branch
+            for t in SERIES_TYPE:
+                setattr(t, func_name, series_func)
+
+    for t in INDEX_TYPE:
+        setattr(t, "agg", aggregate)
+        setattr(t, "aggregate", aggregate)
+        setattr(t, "all", all_index)
+        setattr(t, "any", any_index)
+        setattr(t, "min", min_index)
+        setattr(t, "max", max_index)
+
+
+_install()
+del _install