maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/indexing/loc.py

@@ -28,20 +28,25 @@ from ...tensor.utils import calc_sliced_size, filter_inputs
 from ...utils import is_full_slice, lazy_import, pd_release_version
 from ..core import DATAFRAME_TYPE, IndexValue
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
-from ..utils import parse_index
+from ..utils import parse_index, validate_axis
 from .iloc import DataFrameIlocSetItem
 
 cudf = lazy_import("cudf")
 with_slice_locs_kind = pd_release_version < (1, 4, 0)
 
 
-def process_loc_indexes(inp, indexes, fetch_index: bool = True):
+def process_loc_indexes(inp, indexes, fetch_index: bool = True, axis=None):
     ndim = inp.ndim
 
     if not isinstance(indexes, tuple):
         indexes = (indexes,)
+    if axis is not None and inp.axes[axis].nlevels > 1:
+        indexes = (indexes,)
     if len(indexes) < ndim:
-        indexes += (slice(None),) * (ndim - len(indexes))
+        if axis == 0 or axis is None:
+            indexes += (slice(None),) * (ndim - len(indexes))
+        else:
+            indexes = (slice(None),) * (ndim - len(indexes)) + indexes
     if len(indexes) > ndim:
         raise IndexingError("Too many indexers")
 
@@ -67,8 +72,9 @@ def process_loc_indexes(inp, indexes, fetch_index: bool = True):
 
 
 class DataFrameLoc:
-    def __init__(self, obj):
+    def __init__(self, obj, axis=None):
         self._obj = obj
+        self._axis = axis
 
     def _use_iloc(self, indexes):
         # for RangeIndex from 0, use iloc instead of loc
@@ -105,7 +111,7 @@ class DataFrameLoc:
         return True, None
 
     def __getitem__(self, indexes):
-        indexes = process_loc_indexes(self._obj, indexes)
+        indexes = process_loc_indexes(self._obj, indexes, axis=self._axis)
 
         use_iloc, new_indexes = self._use_iloc(indexes)
         if use_iloc:
@@ -120,7 +126,9 @@ class DataFrameLoc:
             raise NotImplementedError("Only scalar value is supported to set by loc")
         if not isinstance(self._obj, DATAFRAME_TYPE):
             raise NotImplementedError("Only DataFrame is supported to set by loc")
-        indexes = process_loc_indexes(self._obj, indexes, fetch_index=False)
+        indexes = process_loc_indexes(
+            self._obj, indexes, fetch_index=False, axis=self._axis
+        )
         use_iloc, new_indexes = self._use_iloc(indexes)
         if use_iloc:
             op = DataFrameIlocSetItem(indexes=new_indexes, value=value)
@@ -137,9 +145,13 @@ class DataFrameLoc:
             ret = op([self._obj] + indices_tileable)
             self._obj.data = ret.data
 
+    def __call__(self, axis):
+        axis = validate_axis(axis, self._obj)
+        return DataFrameLoc(self._obj, axis)
+
 
 class DataFrameLocSetItem(DataFrameOperator, DataFrameOperatorMixin):
-    _op_type_ = opcodes.DATAFRAME_ILOC_SETITEM
+    _op_type_ = opcodes.DATAFRAME_LOC_SETITEM
 
     indexes = ListField("indexes", default=None)
     value = AnyField("value", default=None)
@@ -336,6 +348,17 @@ class DataFrameLocGetItem(DataFrameOperator, DataFrameOperatorMixin):
                 else:
                     # append None to indicate returning Series
                     param["shape"] = None
+            elif isinstance(index, tuple):
+                has_ranges = any(
+                    isinstance(i, (slice, np.ndarray))
+                    or (hasattr(i, "dtype") and index.ndim == 1)
+                    for i in index
+                )
+                if has_ranges:
+                    param["shape"] = np.nan
+                    param["index_value"] = parse_index(pd_index, inp, index)
+                else:
+                    param["shape"] = None
             else:
                 param["shape"] = None
             return param
@@ -411,4 +434,261 @@ class DataFrameLocGetItem(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def loc(a):
+    """
+    Access a group of rows and columns by label(s) or a boolean array.
+
+    ``.loc[]`` is primarily label based, but may also be used with a
+    boolean array.
+
+    Allowed inputs are:
+
+    - A single label, e.g. ``5`` or ``'a'``, (note that ``5`` is
+      interpreted as a *label* of the index, and **never** as an
+      integer position along the index).
+    - A list or array of labels, e.g. ``['a', 'b', 'c']``.
+    - A slice object with labels, e.g. ``'a':'f'``.
+
+      .. warning:: Note that contrary to usual python slices, **both** the
+          start and the stop are included
+
+    - A boolean array of the same length as the axis being sliced,
+      e.g. ``[True, False, True]``.
+    - An alignable boolean Series. The index of the key will be aligned before
+      masking.
+    - An alignable Index. The Index of the returned selection will be the input.
+    - A ``callable`` function with one argument (the calling Series or
+      DataFrame) and that returns valid output for indexing (one of the above)
+
+    See more at :ref:`Selection by Label <indexing.label>`.
+
+    Raises
+    ------
+    KeyError
+        If any items are not found.
+    IndexingError
+        If an indexed key is passed and its index is unalignable to the frame index.
+
+    See Also
+    --------
+    DataFrame.at : Access a single value for a row/column label pair.
+    DataFrame.iloc : Access group of rows and columns by integer position(s).
+    DataFrame.xs : Returns a cross-section (row(s) or column(s)) from the
+        Series/DataFrame.
+    Series.loc : Access group of values using labels.
+
+    Examples
+    --------
+    **Getting values**
+
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2], [4, 5], [7, 8]],
+    ...      index=['cobra', 'viper', 'sidewinder'],
+    ...      columns=['max_speed', 'shield'])
+    >>> df.execute()
+                max_speed  shield
+    cobra               1       2
+    viper               4       5
+    sidewinder          7       8
+
+    Single label. Note this returns the row as a Series.
+
+    >>> df.loc['viper'].execute()
+    max_speed    4
+    shield       5
+    Name: viper, dtype: int64
+
+    List of labels. Note using ``[[]]`` returns a DataFrame.
+
+    >>> df.loc[['viper', 'sidewinder']].execute()
+                max_speed  shield
+    viper               4       5
+    sidewinder          7       8
+
+    Single label for row and column
+
+    >>> df.loc['cobra', 'shield'].execute()
+    2
+
+    Slice with labels for row and single label for column. As mentioned
+    above, note that both the start and stop of the slice are included.
+
+    >>> df.loc['cobra':'viper', 'max_speed'].execute()
+    cobra    1
+    viper    4
+    Name: max_speed, dtype: int64
+
+    Boolean list with the same length as the row axis
+
+    >>> df.loc[[False, False, True]].execute()
+                max_speed  shield
+    sidewinder          7       8
+
+    Alignable boolean Series:
+
+    >>> df.loc[md.Series([False, True, False],
+    ...        index=['viper', 'sidewinder', 'cobra'])].execute()
+                max_speed  shield
+    sidewinder          7       8
+
+    Index (same behavior as ``df.reindex``)
+
+    >>> df.loc[md.Index(["cobra", "viper"], name="foo")].execute()
+           max_speed  shield
+    foo
+    cobra          1       2
+    viper          4       5
+
+    Conditional that returns a boolean Series
+
+    >>> df.loc[df['shield'] > 6].execute()
+                max_speed  shield
+    sidewinder          7       8
+
+    Conditional that returns a boolean Series with column labels specified
+
+    >>> df.loc[df['shield'] > 6, ['max_speed']].execute()
+                max_speed
+    sidewinder          7
+
+    Callable that returns a boolean Series
+
+    >>> df.loc[lambda df: df['shield'] == 8].execute()
+                max_speed  shield
+    sidewinder          7       8
+
+    **Setting values**
+
+    Set value for all items matching the list of labels
+
+    >>> df.loc[['viper', 'sidewinder'], ['shield']] = 50
+    >>> df.execute()
+                max_speed  shield
+    cobra               1       2
+    viper               4      50
+    sidewinder          7      50
+
+    Set value for an entire row
+
+    >>> df.loc['cobra'] = 10
+    >>> df.execute()
+                max_speed  shield
+    cobra              10      10
+    viper               4      50
+    sidewinder          7      50
+
+    Set value for an entire column
+
+    >>> df.loc[:, 'max_speed'] = 30
+    >>> df.execute()
+                max_speed  shield
+    cobra              30      10
+    viper              30      50
+    sidewinder         30      50
+
+    Set value for rows matching callable condition
+
+    >>> df.loc[df['shield'] > 35] = 0
+    >>> df.execute()
+                max_speed  shield
+    cobra              30      10
+    viper               0       0
+    sidewinder          0       0
+
+    **Getting values on a DataFrame with an index that has integer labels**
+
+    Another example using integers for the index
+
+    >>> df = md.DataFrame([[1, 2], [4, 5], [7, 8]],
+    ...      index=[7, 8, 9], columns=['max_speed', 'shield'])
+    >>> df.execute()
+       max_speed  shield
+    7          1       2
+    8          4       5
+    9          7       8
+
+    Slice with integer labels for rows. As mentioned above, note that both
+    the start and stop of the slice are included.
+
+    >>> df.loc[7:9].execute()
+       max_speed  shield
+    7          1       2
+    8          4       5
+    9          7       8
+
+    **Getting values with a MultiIndex**
+
+    A number of examples using a DataFrame with a MultiIndex
+
+    >>> tuples = [
+    ...    ('cobra', 'mark i'), ('cobra', 'mark ii'),
+    ...    ('sidewinder', 'mark i'), ('sidewinder', 'mark ii'),
+    ...    ('viper', 'mark ii'), ('viper', 'mark iii')
+    ... ]
+    >>> index = md.MultiIndex.from_tuples(tuples)
+    >>> values = [[12, 2], [0, 4], [10, 20],
+    ...         [1, 4], [7, 1], [16, 36]]
+    >>> df = md.DataFrame(values, columns=['max_speed', 'shield'], index=index)
+    >>> df.execute()
+                         max_speed  shield
+    cobra      mark i           12       2
+               mark ii           0       4
+    sidewinder mark i           10      20
+               mark ii           1       4
+    viper      mark ii           7       1
+               mark iii         16      36
+
+    Single label. Note this returns a DataFrame with a single index.
+
+    >>> df.loc['cobra'].execute()
+             max_speed  shield
+    mark i          12       2
+    mark ii          0       4
+
+    Single index tuple. Note this returns a Series.
+
+    >>> df.loc[('cobra', 'mark ii')].execute()
+    max_speed    0
+    shield       4
+    Name: (cobra, mark ii), dtype: int64
+
+    Single label for row and column. Similar to passing in a tuple, this
+    returns a Series.
+
+    >>> df.loc['cobra', 'mark i'].execute()
+    max_speed    12
+    shield        2
+    Name: (cobra, mark i), dtype: int64
+
+    Single tuple. Note using ``[[]]`` returns a DataFrame.
+
+    >>> df.loc[[('cobra', 'mark ii')]].execute()
+                   max_speed  shield
+    cobra mark ii          0       4
+
+    Single tuple for the index with a single label for the column
+
+    >>> df.loc[('cobra', 'mark i'), 'shield'].execute()
+    2
+
+    Slice from index tuple to single label
+
+    >>> df.loc[('cobra', 'mark i'):'viper'].execute()
+                         max_speed  shield
+    cobra      mark i           12       2
+               mark ii           0       4
+    sidewinder mark i           10      20
+               mark ii           1       4
+    viper      mark ii           7       1
+               mark iii         16      36
+
+    Slice from index tuple to index tuple
+
+    >>> df.loc[('cobra', 'mark i'):('viper', 'mark ii')].execute()
+                        max_speed  shield
+    cobra      mark i          12       2
+               mark ii          0       4
+    sidewinder mark i          10      20
+               mark ii          1       4
+    viper      mark ii          7       1
+    """
     return DataFrameLoc(a)

maxframe/dataframe/indexing/reindex.py

@@ -16,6 +16,8 @@ from typing import List
 
 import numpy as np
 
+from ...core.operator import OperatorStage
+
 try:
     import scipy.sparse as sps
 except ImportError:  # pragma: no cover
@@ -31,7 +33,7 @@ from ...serialization.serializables import (
     StringField,
 )
 from ...tensor import tensor as astensor
-from ...utils import lazy_import, pd_release_version
+from ...utils import is_full_slice, lazy_import, pd_release_version
 from ..core import INDEX_TYPE
 from ..core import Index as DataFrameIndexType
 from ..initializer import Index as asindex
@@ -92,12 +94,19 @@ class DataFrameReindex(DataFrameOperator, DataFrameOperatorMixin):
     @classmethod
     def _set_inputs(cls, op: "DataFrameReindex", inputs: List[EntityData]):
         super()._set_inputs(op, inputs)
-        inputs_iter = iter(op._inputs)
+        if getattr(op, "indexes", None):
+            op.index, op.columns = [
+                None if is_full_slice(idx) else idx for idx in list(op.indexes) + [None]
+            ][:2]
+        inputs_iter = iter(inputs)
         op._input = next(inputs_iter)
         if op.index is not None and isinstance(op.index, ENTITY_TYPE):
             op.index = next(inputs_iter)
-        if op.fill_value is not None and isinstance(op.fill_value, ENTITY_TYPE):
-            op.fill_value = next(inputs_iter)
+        if op.fill_value is not None:
+            if op.stage == OperatorStage.agg:
+                op.fill_value = None
+            elif isinstance(op.fill_value, ENTITY_TYPE):
+                op.fill_value = next(inputs_iter)
 
     def __call__(self, df_or_series):
         inputs = [df_or_series]
@@ -363,7 +372,7 @@ def reindex(
     axes_kwargs = dict(index=index, columns=columns, axis=axis)
     axes = validate_axis_style_args(
         df_or_series,
-        (labels,),
+        (labels,) if labels is not None else (),
         {k: v for k, v in axes_kwargs.items() if v is not None},
         "labels",
         "reindex",

maxframe/dataframe/indexing/rename.py

@@ -16,6 +16,7 @@ import warnings
 
 from ... import opcodes
 from ...core import get_output_types
+from ...serialization import PickleContainer
 from ...serialization.serializables import AnyField, StringField
 from ..core import INDEX_TYPE, SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
@@ -34,6 +35,11 @@ class DataFrameRename(DataFrameOperator, DataFrameOperatorMixin):
     def __init__(self, output_types=None, **kw):
         super().__init__(_output_types=output_types, **kw)
 
+    def has_custom_code(self) -> bool:
+        return isinstance(self.columns_mapper, PickleContainer) or isinstance(
+            self.index_mapper, PickleContainer
+        )
+
     def _calc_renamed_df(self, df, errors="ignore"):
         empty_df = build_df(df)
         return empty_df.rename(

maxframe/dataframe/indexing/rename_axis.py

@@ -85,8 +85,8 @@ def rename_axis_with_level(
         else:
             columns = mapper
     op = DataFrameRenameAxis(
-        index=None if index is no_default else index,
-        columns=None if columns is no_default else columns,
+        index=index,
+        columns=columns,
         copy_value=copy,
         level=level,
         axis=0 if index is not no_default else 1,

maxframe/dataframe/indexing/reorder_levels.py

@@ -0,0 +1,143 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ... import opcodes
+from ...core import get_output_types
+from ...serialization.serializables import AnyField, Int32Field
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index, validate_axis
+
+
+class DataFrameReorderLevels(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.REORDER_LEVELS
+
+    order = AnyField("order")
+    axis = Int32Field("axis", default=0)
+
+    def __call__(self, df_or_series):
+        # Determine output type
+        self._output_types = get_output_types(df_or_series)
+
+        if self.axis == 0:
+            src_idx_value = df_or_series.index_value
+        else:
+            src_idx_value = df_or_series.columns_value
+
+        # Create reordered index
+        pd_index = src_idx_value.to_pandas()
+        if not isinstance(pd_index, pd.MultiIndex):
+            raise ValueError("reorder_levels can only be used with MultiIndex")
+        pd_index = pd_index.reorder_levels(self.order)
+
+        params = df_or_series.params
+        if self.axis == 0:
+            params["index_value"] = parse_index(pd_index)
+        else:
+            params["columns_value"] = parse_index(pd_index, store_data=True)
+        return self.new_tileable([df_or_series], **params)
+
+
+def _reorder_levels(df_or_series, order, axis=0):
+    axis = validate_axis(axis, df_or_series)
+    op = DataFrameReorderLevels(order=order, axis=axis)
+    return op(df_or_series)
+
+
+def df_reorder_levels(df, order, axis=0):
+    """
+    Rearrange index levels using input order. May not drop or duplicate levels.
+
+    Parameters
+    ----------
+    order : list of int or list of str
+        List representing new level order. Reference level by number
+        (position) or by key (label).
+    axis : {0 or 'index', 1 or 'columns'}, default 0
+        Where to reorder levels.
+
+    Returns
+    -------
+    DataFrame
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> data = {
+    ...     "class": ["Mammals", "Mammals", "Reptiles"],
+    ...     "diet": ["Omnivore", "Carnivore", "Carnivore"],
+    ...     "species": ["Humans", "Dogs", "Snakes"],
+    ... }
+    >>> df = md.DataFrame(data, columns=["class", "diet", "species"])
+    >>> df = df.set_index(["class", "diet"])
+    >>> df.execute()
+                                      species
+    class      diet
+    Mammals    Omnivore                Humans
+               Carnivore                 Dogs
+    Reptiles   Carnivore               Snakes
+
+    Let's reorder the levels of the index:
+
+    >>> df.reorder_levels(["diet", "class"]).execute()
+                                      species
+    diet      class
+    Omnivore  Mammals                  Humans
+    Carnivore Mammals                    Dogs
+              Reptiles                 Snakes
+    """
+    return _reorder_levels(df, order, axis=axis)
+
+
+def series_reorder_levels(series, order):
+    """
+    Rearrange index levels using input order.
+
+    May not drop or duplicate levels.
+
+    Parameters
+    ----------
+    order : list of int representing new level order
+        Reference level by number or key.
+
+    Returns
+    -------
+    type of caller (new object)
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> arrays = [mt.array(["dog", "dog", "cat", "cat", "bird", "bird"]),
+    ...           mt.array(["white", "black", "white", "black", "white", "black"])]
+    >>> s = md.Series([1, 2, 3, 3, 5, 2], index=arrays)
+    >>> s.execute()
+    dog   white    1
+          black    2
+    cat   white    3
+          black    3
+    bird  white    5
+          black    2
+    dtype: int64
+    >>> s.reorder_levels([1, 0]).execute()
+    white  dog     1
+    black  dog     2
+    white  cat     3
+    black  cat     3
+    white  bird    5
+    black  bird    2
+    dtype: int64
+    """
+    return _reorder_levels(series, order)

maxframe/dataframe/indexing/reset_index.py

@@ -18,10 +18,12 @@ import pandas as pd
 from ... import opcodes
 from ...core import OutputType
 from ...serialization.serializables import AnyField, BoolField
-from ...utils import no_default
+from ...utils import no_default, pd_release_version
 from ..operators import DATAFRAME_TYPE, DataFrameOperator, DataFrameOperatorMixin
 from ..utils import build_empty_df, build_empty_series, parse_index
 
+_reset_index_has_names = pd_release_version >= (1, 5)
+
 
 class DataFrameResetIndex(DataFrameOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.RESET_INDEX
@@ -29,8 +31,10 @@ class DataFrameResetIndex(DataFrameOperator, DataFrameOperatorMixin):
     level = AnyField("level", default=None)
     drop = BoolField("drop", default=False)
     name = AnyField("name", default=None)
-    col_level = AnyField("col_level", default=0)
-    col_fill = AnyField("col_fill", default="")
+    col_level = AnyField("col_level", default=None)
+    col_fill = AnyField("col_fill", default=None)
+    incremental_index = BoolField("incremental_index", default=False)
+    names = AnyField("names", default=None)
 
     def __init__(self, output_types=None, **kwargs):
         super().__init__(_output_types=output_types, **kwargs)
@@ -76,9 +80,26 @@ class DataFrameResetIndex(DataFrameOperator, DataFrameOperatorMixin):
         else:
             empty_df = build_empty_df(a.dtypes)
             empty_df.index = a.index_value.to_pandas()[:0]
-            empty_df = empty_df.reset_index(
-                level=self.level, col_level=self.col_level, col_fill=self.col_fill
-            )
+
+            if self.names and _reset_index_has_names:
+                empty_df = empty_df.reset_index(
+                    level=self.level,
+                    col_level=self.col_level,
+                    col_fill=self.col_fill,
+                    names=self.names,
+                )
+            else:
+                empty_df = empty_df.reset_index(
+                    level=self.level, col_level=self.col_level, col_fill=self.col_fill
+                )
+                if self.names:
+                    names = (
+                        [self.names] if not isinstance(self.names, list) else self.names
+                    )
+                    cols = list(empty_df.columns)
+                    cols[: len(names)] = names
+                    empty_df.columns = pd.Index(cols, name=empty_df.columns.name)
+
             shape = (a.shape[0], len(empty_df.columns))
             columns_value = parse_index(empty_df.columns, store_data=True)
             dtypes = empty_df.dtypes
@@ -105,6 +126,8 @@ def df_reset_index(
     inplace=False,
     col_level=0,
     col_fill="",
+    names=None,
+    incremental_index=False,
 ):
     """
     Reset the index, or a level of it.
@@ -255,6 +278,8 @@ def df_reset_index(
         drop=drop,
         col_level=col_level,
         col_fill=col_fill,
+        names=names,
+        incremental_index=incremental_index,
         output_types=[OutputType.dataframe],
     )
     ret = op(df)
@@ -270,6 +295,7 @@ def series_reset_index(
     drop=False,
     name=no_default,
     inplace=False,
+    incremental_index=False,
 ):
     """
     Generate a new DataFrame or Series with the index reset.
@@ -389,6 +415,7 @@ def series_reset_index(
         level=level,
         drop=drop,
         name=name,
+        incremental_index=incremental_index,
         output_types=[OutputType.series if drop else OutputType.dataframe],
     )
     ret = op(series)

maxframe/dataframe/indexing/sample.py

@@ -25,6 +25,7 @@ from ...serialization.serializables import (
     Float64Field,
     Int8Field,
     Int64Field,
+    KeyField,
 )
 from ...tensor.random import RandomStateField
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
@@ -41,6 +42,11 @@ class DataFrameSample(DataFrameOperator, DataFrameOperatorMixin):
     axis = Int8Field("axis", default=None)
     seed = Int64Field("seed", default=None)
     random_state = RandomStateField("random_state", default=None)
+    always_multinomial = BoolField("always_multinomial", default=None)
+
+    # for chunks
+    # num of instances for chunks
+    chunk_samples = KeyField("chunk_samples", default=None)
 
     def __init__(self, random_state=None, seed=None, **kw):
         if random_state is None:
@@ -54,6 +60,8 @@ class DataFrameSample(DataFrameOperator, DataFrameOperatorMixin):
         next(it)
         if isinstance(op.weights, ENTITY_TYPE):
             op.weights = next(it)
+        if isinstance(op.chunk_samples, ENTITY_TYPE):
+            op.chunk_samples = next(it)
 
     def __call__(self, df):
         params = df.params