maxframe 0.1.0b5__cp38-cp38-macosx_11_0_arm64.whl

maxframe/dataframe/window/rolling.py

@@ -0,0 +1,376 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+
+import pandas as pd
+
+from ... import opcodes
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int32Field,
+    Int64Field,
+    KeyField,
+    StringField,
+    TupleField,
+)
+from ...utils import pd_release_version
+from ..core import DATAFRAME_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_empty_df, build_empty_series, parse_index, validate_axis
+from .core import Window
+
+_window_has_method = pd_release_version >= (1, 3, 0)
+_with_pandas_issue_38908 = pd_release_version == (1, 2, 0)
+
+
+class DataFrameRollingAgg(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.ROLLING_AGG
+
+    input = KeyField("input")
+    window = AnyField("window")
+    min_periods = Int64Field("min_periods")
+    center = BoolField("center")
+    win_type = StringField("win_type")
+    on = StringField("on")
+    axis = Int32Field("axis")
+    closed = StringField("closed")
+    func = AnyField("func")
+    func_args = TupleField("func_args")
+    func_kwargs = DictField("func_kwargs")
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        input_iter = iter(self._inputs)
+        self.input = next(input_iter)
+
+    def __call__(self, rolling):
+        inp = rolling.input
+
+        if isinstance(inp, DATAFRAME_TYPE):
+            pd_index = inp.index_value.to_pandas()
+            empty_df = build_empty_df(inp.dtypes, index=pd_index[:0])
+            params = rolling.params.copy()
+            if params["win_type"] == "freq":
+                params["win_type"] = None
+            if self.func != "count":
+                empty_df = empty_df._get_numeric_data()
+            test_df = empty_df.rolling(**params).agg(self.func)
+            if self.axis == 0:
+                index_value = inp.index_value
+            else:
+                index_value = parse_index(
+                    test_df.index, rolling.params, inp, store_data=False
+                )
+            return self.new_dataframe(
+                [inp],
+                shape=(inp.shape[0], test_df.shape[1]),
+                dtypes=test_df.dtypes,
+                index_value=index_value,
+                columns_value=parse_index(test_df.columns, store_data=True),
+            )
+        else:
+            pd_index = inp.index_value.to_pandas()
+            empty_series = build_empty_series(
+                inp.dtype, index=pd_index[:0], name=inp.name
+            )
+            test_obj = empty_series.rolling(**rolling.params).agg(self.func)
+            if isinstance(test_obj, pd.DataFrame):
+                return self.new_dataframe(
+                    [inp],
+                    shape=(inp.shape[0], test_obj.shape[1]),
+                    dtypes=test_obj.dtypes,
+                    index_value=inp.index_value,
+                    columns_value=parse_index(test_obj.dtypes.index, store_data=True),
+                )
+            else:
+                return self.new_series(
+                    [inp],
+                    shape=inp.shape,
+                    dtype=test_obj.dtype,
+                    index_value=inp.index_value,
+                    name=test_obj.name,
+                )
+
+
+class Rolling(Window):
+    window = AnyField("window", default=None)
+    min_periods = Int64Field("min_periods", default=None)
+    center = BoolField("center", default=None)
+    win_type = StringField("win_type", default=None)
+    on = StringField("on", default=None)
+    axis = Int32Field("axis", default=None)
+    closed = StringField("closed", default=None)
+    method = StringField("method", default="single")
+
+    @property
+    def params(self):
+        p = OrderedDict()
+
+        if not _window_has_method:  # pragma: no cover
+            args = [
+                "window",
+                "min_periods",
+                "center",
+                "win_type",
+                "axis",
+                "on",
+                "closed",
+            ]
+        else:
+            args = [
+                "window",
+                "min_periods",
+                "center",
+                "win_type",
+                "axis",
+                "on",
+                "closed",
+                "method",
+            ]
+
+        for attr in args:
+            p[attr] = getattr(self, attr)
+        return p
+
+    def _repr_name(self):
+        return "Rolling" if self.win_type is None else "Window"
+
+    def validate(self):
+        # leverage pandas itself to do validation
+        pd_index = self.input.index_value.to_pandas()
+        if isinstance(self.input, DATAFRAME_TYPE):
+            empty_obj = build_empty_df(self.input.dtypes, index=pd_index[:0])
+        else:
+            empty_obj = build_empty_series(
+                self.input.dtype, index=pd_index[:0], name=self.input.name
+            )
+        pd_rolling = empty_obj.rolling(**self.params)
+        for k in self.params:
+            # update value according to pandas rolling
+            setattr(self, k, getattr(pd_rolling, k))
+
+    def aggregate(self, func, *args, **kwargs):
+        op = DataFrameRollingAgg(
+            func=func, func_args=args, func_kwargs=kwargs, **self.params
+        )
+        return op(self)
+
+    def agg(self, func, *args, **kwargs):
+        return self.aggregate(func, *args, **kwargs)
+
+    def count(self):
+        return self.aggregate("count")
+
+    def sum(self, *args, **kwargs):
+        return self.aggregate("sum", *args, **kwargs)
+
+    def mean(self, *args, **kwargs):
+        return self.aggregate("mean", *args, **kwargs)
+
+    def median(self, **kwargs):
+        return self.aggregate("median", **kwargs)
+
+    def var(self, ddof=1, *args, **kwargs):
+        return self.aggregate("var", ddof=ddof, *args, **kwargs)
+
+    def std(self, ddof=1, *args, **kwargs):
+        return self.aggregate("std", ddof=ddof, *args, **kwargs)
+
+    def min(self, *args, **kwargs):
+        return self.aggregate("min", *args, **kwargs)
+
+    def max(self, *args, **kwargs):
+        return self.aggregate("max", *args, **kwargs)
+
+    def skew(self, **kwargs):
+        return self.aggregate("skew", **kwargs)
+
+    def kurt(self, **kwargs):
+        return self.aggregate("kurt", **kwargs)
+
+
+def rolling(
+    obj,
+    window,
+    min_periods=None,
+    center=False,
+    win_type=None,
+    on=None,
+    axis=0,
+    closed=None,
+):
+    """
+    Provide rolling window calculations.
+
+    Parameters
+    ----------
+    window : int, or offset
+        Size of the moving window. This is the number of observations used for
+        calculating the statistic. Each window will be a fixed size.
+        If its an offset then this will be the time period of each window. Each
+        window will be a variable sized based on the observations included in
+        the time-period. This is only valid for datetimelike indexes. This is
+        new in 0.19.0
+    min_periods : int, default None
+        Minimum number of observations in window required to have a value
+        (otherwise result is NA). For a window that is specified by an offset,
+        `min_periods` will default to 1. Otherwise, `min_periods` will default
+        to the size of the window.
+    center : bool, default False
+        Set the labels at the center of the window.
+    win_type : str, default None
+        Provide a window type. If ``None``, all points are evenly weighted.
+        See the notes below for further information.
+    on : str, optional
+        For a DataFrame, a datetime-like column on which to calculate the rolling
+        window, rather than the DataFrame's index. Provided integer column is
+        ignored and excluded from result since an integer index is not used to
+        calculate the rolling window.
+    axis : int or str, default 0
+    closed : str, default None
+        Make the interval closed on the 'right', 'left', 'both' or
+        'neither' endpoints.
+        For offset-based windows, it defaults to 'right'.
+        For fixed windows, defaults to 'both'. Remaining cases not implemented
+        for fixed windows.
+
+    Returns
+    -------
+    a Window or Rolling sub-classed for the particular operation
+
+    See Also
+    --------
+    expanding : Provides expanding transformations.
+    ewm : Provides exponential weighted functions.
+
+    Notes
+    -----
+    By default, the result is set to the right edge of the window. This can be
+    changed to the center of the window by setting ``center=True``.
+    To learn more about the offsets & frequency strings, please see `this link
+    <http://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases>`__.
+
+    The recognized win_types are:
+    * ``boxcar``
+    * ``triang``
+    * ``blackman``
+    * ``hamming``
+    * ``bartlett``
+    * ``parzen``
+    * ``bohman``
+    * ``blackmanharris``
+    * ``nuttall``
+    * ``barthann``
+    * ``kaiser`` (needs beta)
+    * ``gaussian`` (needs std)
+    * ``general_gaussian`` (needs power, width)
+    * ``slepian`` (needs width)
+    * ``exponential`` (needs tau), center is set to None.
+
+    If ``win_type=None`` all points are evenly weighted. To learn more about
+    different window types see `scipy.signal window functions
+    <https://docs.scipy.org/doc/scipy/reference/signal.html#window-functions>`__.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'B': [0, 1, 2, np.nan, 4]})
+    >>> df.execute()
+         B
+    0  0.0
+    1  1.0
+    2  2.0
+    3  NaN
+    4  4.0
+
+    Rolling sum with a window length of 2, using the 'triang'
+    window type.
+
+    >>> df.rolling(2, win_type='triang').sum().execute()
+         B
+    0  NaN
+    1  0.5
+    2  1.5
+    3  NaN
+    4  NaN
+
+    Rolling sum with a window length of 2, min_periods defaults
+    to the window length.
+
+    >>> df.rolling(2).sum().execute()
+         B
+    0  NaN
+    1  1.0
+    2  3.0
+    3  NaN
+    4  NaN
+
+    Same as above, but explicitly set the min_periods
+
+    >>> df.rolling(2, min_periods=1).sum().execute()
+         B
+    0  0.0
+    1  1.0
+    2  3.0
+    3  2.0
+    4  4.0
+
+    A ragged (meaning not-a-regular frequency), time-indexed DataFrame
+
+    >>> df = md.DataFrame({'B': [0, 1, 2, np.nan, 4]},
+    >>>                   index = [md.Timestamp('20130101 09:00:00'),
+    >>>                            md.Timestamp('20130101 09:00:02'),
+    >>>                            md.Timestamp('20130101 09:00:03'),
+    >>>                            md.Timestamp('20130101 09:00:05'),
+    >>>                            md.Timestamp('20130101 09:00:06')])
+    >>> df.execute()
+                           B
+    2013-01-01 09:00:00  0.0
+    2013-01-01 09:00:02  1.0
+    2013-01-01 09:00:03  2.0
+    2013-01-01 09:00:05  NaN
+    2013-01-01 09:00:06  4.0
+
+    Contrasting to an integer rolling window, this will roll a variable
+    length window corresponding to the time period.
+    The default for min_periods is 1.
+
+    >>> df.rolling('2s').sum().execute()
+                           B
+    2013-01-01 09:00:00  0.0
+    2013-01-01 09:00:02  1.0
+    2013-01-01 09:00:03  3.0
+    2013-01-01 09:00:05  NaN
+    2013-01-01 09:00:06  4.0
+    """
+    axis = validate_axis(axis, obj)
+    r = Rolling(
+        input=obj,
+        window=window,
+        min_periods=min_periods,
+        center=center,
+        win_type=win_type,
+        on=on,
+        axis=axis,
+        closed=closed,
+    )
+    r.validate()
+    return r

maxframe/dataframe/window/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/window/tests/test_ewm.py

@@ -0,0 +1,70 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from .... import dataframe as md
+
+
+def test_ewm():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df)
+
+    with pytest.raises(ValueError):
+        _ = df2.ewm()
+
+    with pytest.raises(ValueError):
+        _ = df2.ewm(com=2, alpha=0.3)
+
+    assert pytest.approx(df2.ewm(com=1).alpha) == 0.5
+    with pytest.raises(ValueError):
+        _ = df2.ewm(com=-1)
+
+    assert pytest.approx(df2.ewm(span=3).alpha) == 0.5
+    with pytest.raises(ValueError):
+        _ = df2.ewm(span=0)
+
+    assert pytest.approx(df2.ewm(halflife=1).alpha) == 0.5
+    with pytest.raises(ValueError):
+        _ = df2.ewm(halflife=-1)
+
+    with pytest.raises(ValueError):
+        _ = df2.ewm(alpha=2)
+
+    r = df2.ewm(3)
+    expected = df.ewm(3)
+    assert repr(r) == repr(expected)
+
+    r = df2.ewm(alpha=1)
+    assert type(r).__name__ == "Expanding"
+
+
+def test_ewm_agg():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df, chunk_size=3)
+
+    r = df2.ewm(span=3).agg("mean")
+    expected = df.ewm(span=3).agg("mean")
+
+    assert r.shape == df.shape
+    assert r.index_value is df2.index_value
+    pd.testing.assert_index_equal(r.columns_value.to_pandas(), expected.columns)
+    pd.testing.assert_series_equal(r.dtypes, df2.dtypes)
+
+    aggs = ["mean", "var", "std"]
+    for a in aggs:
+        r = getattr(df2.ewm(span=3), a)()
+        assert r.op.func == a

maxframe/dataframe/window/tests/test_expanding.py

@@ -0,0 +1,66 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from .... import dataframe as md
+
+
+def test_expanding():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df)
+
+    with pytest.raises(NotImplementedError):
+        _ = df2.expanding(3, center=True)
+
+    with pytest.raises(NotImplementedError):
+        _ = df2.expanding(3, axis=1)
+
+    r = df2.expanding(3, center=False)
+    expected = df.expanding(3, center=False)
+    assert repr(r) == repr(expected)
+
+    assert "b" in dir(r)
+
+    with pytest.raises(AttributeError):
+        _ = r.d
+
+    with pytest.raises(KeyError):
+        _ = r["d"]
+
+    with pytest.raises(KeyError):
+        _ = r["a", "d"]
+
+    assert "a" not in dir(r.a)
+    assert "c" not in dir(r["a", "b"])
+
+
+def test_expanding_agg():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df, chunk_size=3)
+
+    r = df2.expanding(3).agg("max")
+    expected = df.expanding(3).agg("max")
+
+    assert r.shape == df.shape
+    assert r.index_value is df2.index_value
+    pd.testing.assert_index_equal(r.columns_value.to_pandas(), expected.columns)
+    pd.testing.assert_series_equal(r.dtypes, df2.dtypes)
+
+    aggs = ["sum", "count", "min", "max", "mean", "var", "std"]
+    for a in aggs:
+        r = getattr(df2.expanding(3), a)()
+        assert r.op.func == a

maxframe/dataframe/window/tests/test_rolling.py

@@ -0,0 +1,57 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from .... import dataframe as md
+
+
+def test_rolling():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df)
+
+    r = df2.rolling(3, min_periods=1, center=True, win_type="triang", closed="both")
+    expected = df.rolling(
+        3, min_periods=1, center=True, win_type="triang", closed="both"
+    )
+    assert repr(r) == repr(expected)
+
+    assert "b" in dir(r)
+
+    with pytest.raises(AttributeError):
+        _ = r.d
+
+    with pytest.raises(KeyError):
+        _ = r["d"]
+
+    with pytest.raises(KeyError):
+        _ = r["a", "d"]
+
+    assert "a" not in dir(r.a)
+    assert "c" not in dir(r["a", "b"])
+
+
+def test_rolling_agg():
+    df = pd.DataFrame(np.random.rand(4, 3), columns=list("abc"))
+    df2 = md.DataFrame(df, chunk_size=3)
+
+    r = df2.rolling(3).agg("max")
+    expected = df.rolling(3).agg("max")
+
+    assert r.shape == df.shape
+    assert r.index_value is df2.index_value
+    pd.testing.assert_index_equal(r.columns_value.to_pandas(), expected.columns)
+    pd.testing.assert_series_equal(r.dtypes, df2.dtypes)

maxframe/env.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Maxframe K8S envs
+MAXFRAME_NAMESPACE = "MAXFRAME_NAMESPACE"
+
+# Maxframe Service common envs
+MAXFRAME_SERVICE_PORT = "MAXFRAME_SERVICE_PORT"
+MAXFRAME_SERVICE_PORT_RETRIES = "MAXFRAME_SERVICE_PORT_RETRIES"
+MAXFRAME_SERVICE_LISTEN_ADDRESS = "MAXFRAME_SERVICE_LISTEN_ADDRESS"
+MAXFRAME_SERVICE_LOG_CONFIG_FILE = "MAXFRAME_SERVICE_LOG_CONFIG_FILE"
+MAXFRAME_SERVICE_ALLOW_ORIGIN = "MAXFRAME_SERVICE_ALLOW_ORIGIN"
+MAXFRAME_SERVICE_BASE_URL = "MF_SERVICE_BASE_URL"
+
+# ODPS envs
+ODPS_BEARER_TOKEN = "ODPS_BEARER_TOKEN"
+ODPS_BEARER_TOKEN_FILE = "ODPS_BEARER_TOKEN_FILE"
+ODPS_BEARER_TOKEN_TIMESTAMP_FILE = "ODPS_BEARER_TOKEN_TIMESTAMP_FILE"
+ODPS_PROJECT_NAME = "ODPS_PROJECT_NAME"
+ODPS_ENDPOINT = "ODPS_ENDPOINT"
+ODPS_TUNNEL_ENDPOINT = "ODPS_TUNNEL_ENDPOINT"
+ODPS_STORAGE_API_ENDPOINT = "ODPS_STORAGE_API_ENDPOINT"

maxframe/errors.py

@@ -0,0 +1,21 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class MaxFrameError(Exception):
+    pass
+
+
+class MaxFrameUserError(MaxFrameError):
+    pass