maxframe 0.1.0b5__cp311-cp311-macosx_11_0_arm64.whl

maxframe/dataframe/sort/core.py

@@ -0,0 +1,36 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    FieldTypes,
+    Int32Field,
+    Int64Field,
+    ListField,
+    StringField,
+)
+from ..operators import DataFrameOperator
+
+
+class DataFrameSortOperator(DataFrameOperator):
+    axis = Int32Field("axis")
+    ascending = AnyField("ascending")
+    inplace = BoolField("inplace")
+    kind = StringField("kind")
+    na_position = StringField("na_position")
+    ignore_index = BoolField("ignore_index")
+    parallel_kind = StringField("parallel_kind")
+    psrs_kinds = ListField("psrs_kinds", FieldTypes.string)
+    nrows = Int64Field("nrows", default=None)

maxframe/dataframe/sort/sort_index.py

@@ -0,0 +1,153 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import BoolField, ListField
+from ..operators import DATAFRAME_TYPE, DataFrameOperatorMixin
+from ..utils import parse_index, validate_axis
+from .core import DataFrameSortOperator
+
+
+class DataFrameSortIndex(DataFrameSortOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.SORT_INDEX
+
+    level = ListField("level", default=None)
+    sort_remaining = BoolField("sort_remaining", default=None)
+
+    def _call_dataframe(self, df):
+        if self.ignore_index:
+            index_value = parse_index(pd.RangeIndex(df.shape[0]))
+        else:
+            index_value = df.index_value
+        if self.axis == 0:
+            return self.new_dataframe(
+                [df],
+                shape=df.shape,
+                dtypes=df.dtypes,
+                index_value=index_value,
+                columns_value=df.columns_value,
+            )
+        else:
+            dtypes = df.dtypes.sort_index(ascending=self.ascending)
+            columns_value = parse_index(dtypes.index, store_data=True)
+            return self.new_dataframe(
+                [df],
+                shape=df.shape,
+                dtypes=dtypes,
+                index_value=index_value,
+                columns_value=columns_value,
+            )
+
+    def _call_series(self, series):
+        if self.axis != 0:  # pragma: no cover
+            raise TypeError(f"Invalid axis: {self.axis}")
+        if self.ignore_index:
+            index_value = parse_index(pd.RangeIndex(series.shape[0]))
+        else:
+            index_value = series.index_value
+
+        return self.new_series(
+            [series],
+            shape=series.shape,
+            dtype=series.dtype,
+            index_value=index_value,
+            name=series.name,
+        )
+
+    def __call__(self, a):
+        if isinstance(a, DATAFRAME_TYPE):
+            self.output_types = [OutputType.dataframe]
+            return self._call_dataframe(a)
+        else:
+            self.output_types = [OutputType.series]
+            return self._call_series(a)
+
+
+def sort_index(
+    a,
+    axis=0,
+    level=None,
+    ascending=True,
+    inplace=False,
+    kind="quicksort",
+    na_position="last",
+    sort_remaining=True,
+    ignore_index: bool = False,
+    parallel_kind="PSRS",
+    psrs_kinds=None,
+):
+    """
+    Sort object by labels (along an axis).
+
+    Parameters
+    ----------
+    a : Input DataFrame or Series.
+    axis : {0 or 'index', 1 or 'columns'}, default 0
+        The axis along which to sort.  The value 0 identifies the rows,
+        and 1 identifies the columns.
+    level : int or level name or list of ints or list of level names
+        If not None, sort on values in specified index level(s).
+    ascending : bool, default True
+        Sort ascending vs. descending.
+    inplace : bool, default False
+        If True, perform operation in-place.
+    kind : {'quicksort', 'mergesort', 'heapsort'}, default 'quicksort'
+        Choice of sorting algorithm. See also ndarray.np.sort for more
+        information.  `mergesort` is the only stable algorithm. For
+        DataFrames, this option is only applied when sorting on a single
+        column or label.
+    na_position : {'first', 'last'}, default 'last'
+        Puts NaNs at the beginning if `first`; `last` puts NaNs at the end.
+        Not implemented for MultiIndex.
+    sort_remaining : bool, default True
+        If True and sorting by level and index is multilevel, sort by other
+        levels too (in order) after sorting by specified level.
+    ignore_index : bool, default False
+        If True, the resulting axis will be labeled 0, 1, …, n - 1.
+    parallel_kind: {'PSRS'}, optional.
+        Parallel sorting algorithm, for the details, refer to:
+        http://csweb.cs.wfu.edu/bigiron/LittleFE-PSRS/build/html/PSRSalgorithm.html
+    psrs_kinds: Sorting algorithms during PSRS algorithm.
+
+    Returns
+    -------
+    sorted_obj : DataFrame or None
+        DataFrame with sorted index if inplace=False, None otherwise.
+    """
+    if na_position not in ["last", "first"]:  # pragma: no cover
+        raise TypeError(f"Invalid na_position: {na_position}")
+    # psrs_kinds = _validate_sort_psrs_kinds(psrs_kinds)
+    axis = validate_axis(axis, a)
+    level = level if isinstance(level, (list, tuple)) or level is None else [level]
+    op = DataFrameSortIndex(
+        level=level,
+        axis=axis,
+        ascending=ascending,
+        inplace=inplace,
+        kind=kind,
+        na_position=na_position,
+        sort_remaining=sort_remaining,
+        ignore_index=ignore_index,
+        parallel_kind=parallel_kind,
+        psrs_kinds=psrs_kinds,
+        gpu=a.op.is_gpu(),
+    )
+    sorted_a = op(a)
+    if inplace:
+        a.data = sorted_a.data
+    else:
+        return sorted_a

maxframe/dataframe/sort/sort_values.py

@@ -0,0 +1,311 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import ListField
+from ..core import IndexValue
+from ..operators import DataFrameOperatorMixin
+from ..utils import parse_index, validate_axis
+from .core import DataFrameSortOperator
+
+
+class DataFrameSortValues(DataFrameSortOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.SORT_VALUES
+
+    by = ListField("by", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def __call__(self, a):
+        assert self.axis == 0
+        if self.ignore_index:
+            index_value = parse_index(pd.RangeIndex(a.shape[0]))
+        else:
+            if isinstance(a.index_value.value, IndexValue.RangeIndex):
+                index_value = parse_index(pd.Index([], dtype=np.int64))
+            else:
+                index_value = a.index_value
+        if a.ndim == 2:
+            return self.new_dataframe(
+                [a],
+                shape=a.shape,
+                dtypes=a.dtypes,
+                index_value=index_value,
+                columns_value=a.columns_value,
+            )
+        else:
+            return self.new_series(
+                [a], shape=a.shape, dtype=a.dtype, index_value=index_value, name=a.name
+            )
+
+
+def dataframe_sort_values(
+    df,
+    by,
+    axis=0,
+    ascending=True,
+    inplace=False,
+    kind="quicksort",
+    na_position="last",
+    ignore_index=False,
+    parallel_kind="PSRS",
+    psrs_kinds=None,
+):
+    """
+    Sort by the values along either axis.
+
+    Parameters
+    ----------
+    df : MaxFrame DataFrame
+         Input dataframe.
+    by : str
+         Name or list of names to sort by.
+    axis : %(axes_single_arg)s, default 0
+         Axis to be sorted.
+    ascending : bool or list of bool, default True
+         Sort ascending vs. descending. Specify list for multiple sort
+         orders.  If this is a list of bools, must match the length of
+         the by.
+    inplace : bool, default False
+         If True, perform operation in-place.
+    kind : {'quicksort', 'mergesort', 'heapsort'}, default 'quicksort'
+         Choice of sorting algorithm. See also ndarray.np.sort for more
+         information.  `mergesort` is the only stable algorithm. For
+         DataFrames, this option is only applied when sorting on a single
+         column or label.
+    na_position : {'first', 'last'}, default 'last'
+         Puts NaNs at the beginning if `first`; `last` puts NaNs at the
+         end.
+    ignore_index : bool, default False
+         If True, the resulting axis will be labeled 0, 1, …, n - 1.
+    parallel_kind : {'PSRS'}, default 'PSRS'
+         Parallel sorting algorithm, for the details, refer to:
+         http://csweb.cs.wfu.edu/bigiron/LittleFE-PSRS/build/html/PSRSalgorithm.html
+
+    Returns
+    -------
+    sorted_obj : DataFrame or None
+        DataFrame with sorted values if inplace=False, None otherwise.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({
+    ...     'col1': ['A', 'A', 'B', np.nan, 'D', 'C'],
+    ...     'col2': [2, 1, 9, 8, 7, 4],
+    ...     'col3': [0, 1, 9, 4, 2, 3],
+    ... })
+    >>> df.execute()
+        col1 col2 col3
+    0   A    2    0
+    1   A    1    1
+    2   B    9    9
+    3   NaN  8    4
+    4   D    7    2
+    5   C    4    3
+
+    Sort by col1
+
+    >>> df.sort_values(by=['col1']).execute()
+        col1 col2 col3
+    0   A    2    0
+    1   A    1    1
+    2   B    9    9
+    5   C    4    3
+    4   D    7    2
+    3   NaN  8    4
+
+    Sort by multiple columns
+
+    >>> df.sort_values(by=['col1', 'col2']).execute()
+        col1 col2 col3
+    1   A    1    1
+    0   A    2    0
+    2   B    9    9
+    5   C    4    3
+    4   D    7    2
+    3   NaN  8    4
+
+    Sort Descending
+
+    >>> df.sort_values(by='col1', ascending=False).execute()
+        col1 col2 col3
+    4   D    7    2
+    5   C    4    3
+    2   B    9    9
+    0   A    2    0
+    1   A    1    1
+    3   NaN  8    4
+
+    Putting NAs first
+
+    >>> df.sort_values(by='col1', ascending=False, na_position='first').execute()
+        col1 col2 col3
+    3   NaN  8    4
+    4   D    7    2
+    5   C    4    3
+    2   B    9    9
+    0   A    2    0
+    1   A    1    1
+    """
+
+    if na_position not in ["last", "first"]:  # pragma: no cover
+        raise TypeError(f"invalid na_position: {na_position}")
+    axis = validate_axis(axis, df)
+    if axis != 0:
+        raise NotImplementedError("Only support sort on axis 0")
+    # psrs_kinds = _validate_sort_psrs_kinds(psrs_kinds)
+    by = by if isinstance(by, (list, tuple)) else [by]
+    if isinstance(ascending, list):  # pragma: no cover
+        if all(ascending):
+            # all are True, convert to True
+            ascending = True
+        elif not any(ascending):
+            # all are False, convert to False
+            ascending = False
+    op = DataFrameSortValues(
+        by=by,
+        axis=axis,
+        ascending=ascending,
+        inplace=inplace,
+        kind=kind,
+        na_position=na_position,
+        ignore_index=ignore_index,
+        parallel_kind=parallel_kind,
+        psrs_kinds=psrs_kinds,
+        gpu=df.op.is_gpu(),
+        output_types=[OutputType.dataframe],
+    )
+    sorted_df = op(df)
+    if inplace:
+        df.data = sorted_df.data
+    else:
+        return sorted_df
+
+
+def series_sort_values(
+    series,
+    axis=0,
+    ascending=True,
+    inplace=False,
+    kind="quicksort",
+    na_position="last",
+    ignore_index=False,
+    parallel_kind="PSRS",
+    psrs_kinds=None,
+):
+    """
+    Sort by the values.
+
+    Sort a Series in ascending or descending order by some
+    criterion.
+
+    Parameters
+    ----------
+    series : input Series.
+    axis : {0 or 'index'}, default 0
+        Axis to direct sorting. The value 'index' is accepted for
+        compatibility with DataFrame.sort_values.
+    ascending : bool, default True
+        If True, sort values in ascending order, otherwise descending.
+    inplace : bool, default False
+        If True, perform operation in-place.
+    kind : {'quicksort', 'mergesort' or 'heapsort'}, default 'quicksort'
+        Choice of sorting algorithm. See also :func:`numpy.sort` for more
+        information. 'mergesort' is the only stable  algorithm.
+    na_position : {'first' or 'last'}, default 'last'
+        Argument 'first' puts NaNs at the beginning, 'last' puts NaNs at
+        the end.
+    ignore_index : bool, default False
+         If True, the resulting axis will be labeled 0, 1, …, n - 1.
+
+    Returns
+    -------
+    Series
+        Series ordered by values.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> raw = pd.Series([np.nan, 1, 3, 10, 5])
+    >>> s = md.Series(raw)
+    >>> s.execute()
+    0     NaN
+    1     1.0
+    2     3.0
+    3     10.0
+    4     5.0
+    dtype: float64
+
+    Sort values ascending order (default behaviour)
+
+    >>> s.sort_values(ascending=True).execute()
+    1     1.0
+    2     3.0
+    4     5.0
+    3    10.0
+    0     NaN
+    dtype: float64
+
+    Sort values descending order
+
+    >>> s.sort_values(ascending=False).execute()
+    3    10.0
+    4     5.0
+    2     3.0
+    1     1.0
+    0     NaN
+    dtype: float64
+
+    Sort values inplace
+
+    >>> s.sort_values(ascending=False, inplace=True)
+    >>> s.execute()
+    3    10.0
+    4     5.0
+    2     3.0
+    1     1.0
+    0     NaN
+    dtype: float64
+
+    Sort values putting NAs first
+    """
+    if na_position not in ["last", "first"]:  # pragma: no cover
+        raise TypeError(f"invalid na_position: {na_position}")
+    axis = validate_axis(axis, series)
+    if axis != 0:
+        raise NotImplementedError("Only support sort on axis 0")
+    # psrs_kinds = _validate_sort_psrs_kinds(psrs_kinds)
+    op = DataFrameSortValues(
+        axis=axis,
+        ascending=ascending,
+        inplace=inplace,
+        kind=kind,
+        na_position=na_position,
+        ignore_index=ignore_index,
+        parallel_kind=parallel_kind,
+        psrs_kinds=psrs_kinds,
+        output_types=[OutputType.series],
+        gpu=series.op.is_gpu(),
+    )
+    sorted_series = op(series)
+    if inplace:
+        series.data = sorted_series.data
+    else:
+        return sorted_series

maxframe/dataframe/sort/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/sort/tests/test_sort.py

@@ -0,0 +1,81 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ...initializer import DataFrame
+from ..sort_index import DataFrameSortIndex, sort_index
+from ..sort_values import DataFrameSortValues, dataframe_sort_values
+
+
+def test_sort_values():
+    raw = pd.DataFrame(
+        {
+            "a": np.random.rand(10),
+            "b": np.random.randint(1000, size=10),
+            "c": np.random.rand(10),
+            "d": [np.random.bytes(10) for _ in range(10)],
+            "e": [pd.Timestamp(f"201{i}") for i in range(10)],
+            "f": [pd.Timedelta(f"{i} days") for i in range(10)],
+        },
+    )
+    df = DataFrame(raw)
+    sorted_df = dataframe_sort_values(df, by="c")
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortValues)
+
+    df = DataFrame(raw, chunk_size=6)
+    sorted_df = dataframe_sort_values(df, by="c")
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortValues)
+
+    df = DataFrame(raw, chunk_size=3)
+    sorted_df = dataframe_sort_values(df, by=["a", "c"])
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortValues)
+
+
+def test_sort_index():
+    raw = pd.DataFrame(
+        np.random.rand(10, 10), columns=np.random.rand(10), index=np.random.rand(10)
+    )
+    df = DataFrame(raw)
+    sorted_df = sort_index(df)
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortIndex)
+    assert sorted_df.op.level is None
+
+    df = DataFrame(raw, chunk_size=6)
+    sorted_df = sort_index(df)
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortIndex)
+
+    df = DataFrame(raw, chunk_size=3)
+    sorted_df = sort_index(df)
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortIndex)
+
+    # support on axis 1
+    df = DataFrame(raw, chunk_size=4)
+    sorted_df = sort_index(df, axis=1)
+
+    assert sorted_df.shape == raw.shape
+    assert isinstance(sorted_df.op, DataFrameSortIndex)

maxframe/dataframe/statistics/__init__.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def _install():
+    from ..core import DATAFRAME_TYPE, SERIES_TYPE
+    from .corr import df_corr, df_corrwith, series_autocorr, series_corr
+    from .quantile import quantile_dataframe, quantile_series
+
+    for t in SERIES_TYPE:
+        t.quantile = quantile_series
+        t.corr = series_corr
+        t.autocorr = series_autocorr
+
+    for t in DATAFRAME_TYPE:
+        t.quantile = quantile_dataframe
+        t.corr = df_corr
+        t.corrwith = df_corrwith
+
+
+_install()
+del _install