maxframe 2.3.0__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

maxframe/dataframe/misc/memory_usage.py

@@ -0,0 +1,248 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import BoolField, Int64Field
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index
+
+
+class DataFrameMemoryUsage(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.MEMORY_USAGE
+
+    # raw arguments of memory_usage method
+    index = BoolField("index", default=None)
+    deep = BoolField("deep", default=None)
+
+    # size of range index, when set, the value will be prepended to the result series
+    # if the input is a dataframe, or added to the result when the input is a series
+    range_index_size = Int64Field("range_index_size", default=None)
+
+    def _adapt_index(self, input_index, index=0):
+        """
+        When ``index=True`` is passed, an extra column will be prepended to the result series
+        Thus we need to update the index of initial chunk for returned dataframe chunks
+        """
+        if not self.index or index != 0:
+            return input_index
+        idx_data = input_index.to_pandas().insert(0, "Index")
+        return parse_index(idx_data, store_data=True)
+
+    def _adapt_nsplits(self, input_nsplit):
+        """
+        When ``index=True`` is passed, the size of returned series is one element larger
+        than the number of columns, which affects ``nsplits``.
+        """
+        if not self.index:
+            return (input_nsplit[-1],)
+        nsplits_list = list(input_nsplit[-1])
+        nsplits_list[0] += 1
+        return (tuple(nsplits_list),)
+
+    def __call__(self, df_or_series):
+        """
+        Return output object of memory_usage() call
+        """
+        if df_or_series.ndim == 1:
+            # the input data is a series, a Scalar will be returned
+            return self.new_scalar([df_or_series], dtype=np.dtype(int))
+        else:
+            # the input data is a DataFrame, a Scalar will be returned
+            # calculate shape of returning series given ``op.index``
+            new_shape = (
+                (df_or_series.shape[-1] + 1,)
+                if self.index
+                else (df_or_series.shape[-1],)
+            )
+            return self.new_series(
+                [df_or_series],
+                index_value=self._adapt_index(df_or_series.columns_value),
+                shape=new_shape,
+                dtype=np.dtype(int),
+            )
+
+
+def df_memory_usage(df, index=True, deep=False):
+    """
+    Return the memory usage of each column in bytes.
+
+    The memory usage can optionally include the contribution of
+    the index and elements of `object` dtype.
+
+    This value is displayed in `DataFrame.info` by default. This can be
+    suppressed by setting ``pandas.options.display.memory_usage`` to False.
+
+    Parameters
+    ----------
+    index : bool, default True
+        Specifies whether to include the memory usage of the DataFrame's
+        index in returned Series. If ``index=True``, the memory usage of
+        the index is the first item in the output.
+    deep : bool, default False
+        If True, introspect the data deeply by interrogating
+        `object` dtypes for system-level memory consumption, and include
+        it in the returned values.
+
+    Returns
+    -------
+    Series
+        A Series whose index is the original column names and whose values
+        is the memory usage of each column in bytes.
+
+    See Also
+    --------
+    numpy.ndarray.nbytes : Total bytes consumed by the elements of an
+        ndarray.
+    Series.memory_usage : Bytes consumed by a Series.
+    Categorical : Memory-efficient array for string values with
+        many repeated values.
+    DataFrame.info : Concise summary of a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> dtypes = ['int64', 'float64', 'complex128', 'object', 'bool']
+    >>> data = dict([(t, mt.ones(shape=5000).astype(t))
+    ...              for t in dtypes])
+    >>> df = md.DataFrame(data)
+    >>> df.head().execute()
+       int64  float64            complex128  object  bool
+    0      1      1.0    1.000000+0.000000j       1  True
+    1      1      1.0    1.000000+0.000000j       1  True
+    2      1      1.0    1.000000+0.000000j       1  True
+    3      1      1.0    1.000000+0.000000j       1  True
+    4      1      1.0    1.000000+0.000000j       1  True
+
+    >>> df.memory_usage().execute()
+    Index           128
+    int64         40000
+    float64       40000
+    complex128    80000
+    object        40000
+    bool           5000
+    dtype: int64
+
+    >>> df.memory_usage(index=False).execute()
+    int64         40000
+    float64       40000
+    complex128    80000
+    object        40000
+    bool           5000
+    dtype: int64
+
+    The memory footprint of `object` dtype columns is ignored by default:
+
+    >>> df.memory_usage(deep=True).execute()
+    Index            128
+    int64          40000
+    float64        40000
+    complex128     80000
+    object        160000
+    bool            5000
+    dtype: int64
+
+    Use a Categorical for efficient storage of an object-dtype column with
+    many repeated values.
+
+    >>> df['object'].astype('category').memory_usage(deep=True).execute()
+    5216
+    """
+    op = DataFrameMemoryUsage(index=index, deep=deep)
+    return op(df)
+
+
+def series_memory_usage(series, index=True, deep=False):
+    """
+    Return the memory usage of the Series.
+
+    The memory usage can optionally include the contribution of
+    the index and of elements of `object` dtype.
+
+    Parameters
+    ----------
+    index : bool, default True
+        Specifies whether to include the memory usage of the Series index.
+    deep : bool, default False
+        If True, introspect the data deeply by interrogating
+        `object` dtypes for system-level memory consumption, and include
+        it in the returned value.
+
+    Returns
+    -------
+    int
+        Bytes of memory consumed.
+
+    See Also
+    --------
+    numpy.ndarray.nbytes : Total bytes consumed by the elements of the
+        array.
+    DataFrame.memory_usage : Bytes consumed by a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(range(3))
+    >>> s.memory_usage().execute()
+    152
+
+    Not including the index gives the size of the rest of the data, which
+    is necessarily smaller:
+
+    >>> s.memory_usage(index=False).execute()
+    24
+
+    The memory footprint of `object` values is ignored by default:
+
+    >>> s = md.Series(["a", "b"])
+    >>> s.values.execute()
+    array(['a', 'b'], dtype=object)
+
+    >>> s.memory_usage().execute()
+    144
+
+    >>> s.memory_usage(deep=True).execute()
+    260
+    """
+    op = DataFrameMemoryUsage(index=index, deep=deep)
+    return op(series)
+
+
+def index_memory_usage(index, deep=False):
+    """
+    Memory usage of the values.
+
+    Parameters
+    ----------
+    deep : bool
+        Introspect the data deeply, interrogate
+        `object` dtypes for system-level memory consumption.
+
+    Returns
+    -------
+    bytes used
+
+    See Also
+    --------
+    numpy.ndarray.nbytes
+
+    Notes
+    -----
+    Memory usage does not include memory consumed by elements that
+    are not components of the array if deep=False
+    """
+    op = DataFrameMemoryUsage(index=False, deep=deep)
+    return op(index)

maxframe/dataframe/misc/pct_change.py

@@ -0,0 +1,68 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..utils import validate_axis
+
+
+def pct_change(
+    df_or_series, periods=1, fill_method="pad", limit=None, freq=None, **kwargs
+):
+    # FIXME: https://github.com/aliyun/alibabacloud-odps-maxframe-client/issues/32
+    """
+    Percentage change between the current and a prior element.
+
+    Computes the percentage change from the immediately previous row by
+    default. This is useful in comparing the percentage of change in a time
+    series of elements.
+
+    Parameters
+    ----------
+    periods : int, default 1
+        Periods to shift for forming percent change.
+    fill_method : str, default 'pad'
+        How to handle NAs before computing percent changes.
+    limit : int, default None
+        The number of consecutive NAs to fill before stopping.
+    freq : DateOffset, timedelta, or str, optional
+        Increment to use from time series API (e.g. 'M' or BDay()).
+    **kwargs
+        Additional keyword arguments are passed into
+        `DataFrame.shift` or `Series.shift`.
+
+    Returns
+    -------
+    chg : Series or DataFrame
+        The same type as the calling object.
+
+    See Also
+    --------
+    Series.diff : Compute the difference of two elements in a Series.
+    DataFrame.diff : Compute the difference of two elements in a DataFrame.
+    Series.shift : Shift the index by some number of periods.
+    DataFrame.shift : Shift the index by some number of periods.
+    """
+
+    axis = validate_axis(kwargs.pop("axis", 0))
+    if fill_method is None:
+        data = df_or_series
+    else:
+        data = df_or_series.fillna(method=fill_method, axis=axis, limit=limit)
+
+    rs = data.div(data.shift(periods=periods, freq=freq, axis=axis, **kwargs)) - 1
+    if freq is not None:
+        # Shift method is implemented differently when freq is not None
+        # We want to restore the original index
+        rs = rs.loc[~rs.index.duplicated()]
+        rs = rs.reindex_like(data)
+    return rs

maxframe/dataframe/misc/qcut.py

@@ -0,0 +1,104 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_integer
+
+from ...core import ENTITY_TYPE
+from ...tensor import tensor as astensor
+from ...tensor.statistics.percentile import percentile
+from ..core import DATAFRAME_TYPE, SERIES_TYPE
+from ..initializer import DataFrame, Series
+from .cut import cut
+
+
+def qcut(x, q, labels=None, retbins=False, precision=3, duplicate="raise"):
+    """
+    Quantile-based discretization function.
+
+    Discretize variable into equal-sized buckets based on rank or based
+    on sample quantiles. For example 1000 values for 10 quantiles would
+    produce a Categorical object indicating quantile membership for each data point.
+
+    Parameters
+    ----------
+    x : 1d tensor or Series
+    q : int or list-like of float
+        Number of quantiles. 10 for deciles, 4 for quartiles, etc. Alternately
+        array of quantiles, e.g. [0, .25, .5, .75, 1.] for quartiles.
+    labels : array or False, default None
+        Used as labels for the resulting bins. Must be of the same length as
+        the resulting bins. If False, return only integer indicators of the
+        bins. If True, raises an error.
+    retbins : bool, optional
+        Whether to return the (bins, labels) or not. Can be useful if bins
+        is given as a scalar.
+    precision : int, optional
+        The precision at which to store and display the bins labels.
+    duplicates : {default 'raise', 'drop'}, optional
+        If bin edges are not unique, raise ValueError or drop non-uniques.
+
+    Returns
+    -------
+    out : Categorical or Series or tensor of integers if labels is False
+        The return type (Categorical or Series) depends on the input: a Series
+        of type category if input is a Series else Categorical. Bins are
+        represented as categories when categorical data is returned.
+    bins : tensor of floats
+        Returned only if `retbins` is True.
+
+    Notes
+    -----
+    Out of bounds values will be NA in the resulting Categorical object
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> md.qcut(range(5), 4).execute()
+    ... # doctest: +ELLIPSIS
+    [(-0.001, 1.0], (-0.001, 1.0], (1.0, 2.0], (2.0, 3.0], (3.0, 4.0]]
+    Categories (4, interval[float64]): [(-0.001, 1.0] < (1.0, 2.0] ...
+
+    >>> md.qcut(range(5), 3, labels=["good", "medium", "bad"]).execute()
+    ... # doctest: +SKIP
+    [good, good, medium, bad, bad]
+    Categories (3, object): [good < medium < bad]
+
+    >>> md.qcut(range(5), 4, labels=False).execute()
+    array([0, 0, 1, 2, 3])
+    """
+    if is_integer(q):
+        q = np.linspace(0, 1, q + 1)
+
+    if isinstance(x, (DATAFRAME_TYPE, SERIES_TYPE, pd.DataFrame, pd.Series)):
+        x = DataFrame(x) if x.ndim == 2 else Series(x)
+        bins = x.quantile(q)
+    else:
+        x = astensor(x)
+        if isinstance(q, ENTITY_TYPE):
+            q = q * 100
+        else:
+            q = [iq * 100 for iq in q]
+        bins = percentile(x, q)
+
+    return cut(
+        x,
+        bins,
+        labels=labels,
+        retbins=retbins,
+        precision=precision,
+        include_lowest=True,
+        duplicates=duplicate,
+    )

maxframe/dataframe/misc/rechunk.py

@@ -0,0 +1,59 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import AnyField
+from ...tensor.rechunk.rechunk import chunk_size_type
+from ...typing_ import TileableType
+from ..core import DATAFRAME_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameRechunk(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.RECHUNK
+
+    chunk_size = AnyField("chunk_size")
+
+    def __call__(self, x):
+        if isinstance(x, DATAFRAME_TYPE):
+            return self.new_dataframe(
+                [x],
+                shape=x.shape,
+                dtypes=x.dtypes,
+                columns_value=x.columns_value,
+                index_value=x.index_value,
+            )
+        else:
+            self.output_types = x.op.output_types
+            f = (
+                self.new_series
+                if self.output_types[0] == OutputType.series
+                else self.new_index
+            )
+            return f(
+                [x],
+                shape=x.shape,
+                dtype=x.dtype,
+                index_value=x.index_value,
+                name=x.name,
+            )
+
+
+def rechunk(a: TileableType, chunk_size: chunk_size_type, reassign_worker=False):
+    op = DataFrameRechunk(
+        chunk_size=chunk_size,
+        reassign_worker=reassign_worker,
+    )
+    return op(a)

maxframe/dataframe/misc/repeat.py

@@ -0,0 +1,159 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+from pandas.api.types import is_list_like
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, EntityData, get_output_types
+from ...serialization.serializables import AnyField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index, validate_axis
+
+
+class DataFrameRepeat(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.REPEAT
+
+    repeats = AnyField("repeats", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameRepeat", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if isinstance(op.repeats, ENTITY_TYPE):
+            op.repeats = inputs[1]
+
+    def __call__(self, obj, repeats):
+        self._output_types = get_output_types(obj)
+        test_index = obj.index_value.to_pandas()[:0]
+
+        params = obj.params
+        params["index_value"] = parse_index(test_index, obj, type(self), self.repeats)
+        params["shape"] = (np.nan,)
+
+        inputs = [obj]
+        if isinstance(repeats, ENTITY_TYPE):
+            inputs.append(repeats)
+        return self.new_tileable(inputs, **params)
+
+
+def _repeat(obj, repeats, axis=None):
+    from ...tensor.datasource import tensor
+
+    axis = validate_axis(axis or 0, obj)
+    if is_list_like(repeats):
+        repeats = tensor(repeats)
+    op = DataFrameRepeat(repeats=repeats, axis=axis)
+    return op(obj, repeats)
+
+
+def series_repeat(obj, repeats, axis=None):
+    """
+    Repeat elements of a Series.
+
+    Returns a new Series where each element of the current Series
+    is repeated consecutively a given number of times.
+
+    Parameters
+    ----------
+    repeats : int or array of ints
+        The number of repetitions for each element. This should be a
+        non-negative integer. Repeating 0 times will return an empty
+        Series.
+    axis : None
+        Must be ``None``. Has no effect but is accepted for compatibility
+        with numpy.
+
+    Returns
+    -------
+    Series
+        Newly created Series with repeated elements.
+
+    See Also
+    --------
+    Index.repeat : Equivalent function for Index.
+    numpy.repeat : Similar method for :class:`numpy.ndarray`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(['a', 'b', 'c'])
+    >>> s.execute()
+    0    a
+    1    b
+    2    c
+    dtype: object
+    >>> s.repeat(2).execute()
+    0    a
+    0    a
+    1    b
+    1    b
+    2    c
+    2    c
+    dtype: object
+    >>> s.repeat([1, 2, 3]).execute()
+    0    a
+    1    b
+    1    b
+    2    c
+    2    c
+    2    c
+    dtype: object
+    """
+    return _repeat(obj, repeats, axis=axis)
+
+
+def index_repeat(obj, repeats, axis=None):
+    """
+    Repeat elements of an Index.
+
+    Returns a new Index where each element of the current Index
+    is repeated consecutively a given number of times.
+
+    Parameters
+    ----------
+    repeats : int or array of ints
+        The number of repetitions for each element. This should be a
+        non-negative integer. Repeating 0 times will return an empty
+        Index.
+    axis : None
+        Must be ``None``. Has no effect but is accepted for compatibility
+        with numpy.
+
+    Returns
+    -------
+    repeated_index : Index
+        Newly created Index with repeated elements.
+
+    See Also
+    --------
+    Series.repeat : Equivalent function for Series.
+    numpy.repeat : Similar method for :class:`numpy.ndarray`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> idx = md.Index(['a', 'b', 'c'])
+    >>> idx.execute()
+    Index(['a', 'b', 'c'], dtype='object')
+    >>> idx.repeat(2).execute()
+    Index(['a', 'a', 'b', 'b', 'c', 'c'], dtype='object')
+    >>> idx.repeat([1, 2, 3]).execute()
+    Index(['a', 'b', 'b', 'c', 'c', 'c'], dtype='object')
+    """
+    return _repeat(obj, repeats, axis=axis)