maxframe 1.3.0__cp310-cp310-macosx_10_9_universal2.whl → 2.0.0__cp310-cp310-macosx_10_9_universal2.whl

maxframe/dataframe/misc/get_dummies.py

@@ -12,11 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
+
 import numpy as np
 import pandas as pd
 
-from ...core import OutputType
-from ...serialization.serializables import AnyField, BoolField, ListField, StringField
+from ...core import EntityData, OutputType
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    KeyField,
+    ListField,
+    StringField,
+)
 from ..datasource.dataframe import from_pandas as from_pandas_df
 from ..datasource.series import from_pandas as from_pandas_series
 from ..initializer import Series as asseries
@@ -34,55 +42,26 @@ class DataFrameGetDummies(DataFrameOperator, DataFrameOperatorMixin):
     drop_first = BoolField("drop_first", default=None)
     dtype = AnyField("dtype", default=None)
 
-    def __init__(self, **kws):
-        super().__init__(**kws)
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
         self.output_types = [OutputType.dataframe]
 
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameGetDummies", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:  # pragma: no branch
+            op.agg_results = inputs[-1]
+
     def __call__(self, data):
-        if isinstance(data, (list, tuple)):
-            data = asseries(data)
-        elif isinstance(data, pd.Series):
-            data = from_pandas_series(data)
-        elif isinstance(data, pd.DataFrame):
-            data = from_pandas_df(data)
-
-        if self.prefix is not None:
-            if isinstance(self.prefix, list):
-                if self.columns is not None:
-                    encoding_col_num = len(self.columns)
-                else:
-                    encoding_col_num = 0
-                    for dtype in data.dtypes.values:
-                        if dtype.kind in _encoding_dtype_kind:
-                            encoding_col_num += 1
-                prefix_num = len(self.prefix)
-                if prefix_num != encoding_col_num:
-                    raise ValueError(
-                        f"Length of 'prefix' ({prefix_num}) did not match "
-                        + f"the length of the columns being encoded ({encoding_col_num})"
-                    )
-            elif isinstance(self.prefix, dict):
-                if self.columns is not None:
-                    encoding_col_num = len(self.columns)
-                    prefix_num = len(self.prefix)
-                    if prefix_num != encoding_col_num:
-                        raise ValueError(
-                            f"Length of 'prefix' ({prefix_num}) did not match "
-                            + f"the length of the columns being encoded ({encoding_col_num})"
-                        )
-                    columns = self.prefix.keys()
-                    for columns_columnname, prefix_columnname in zip(
-                        columns, list(self.columns)
-                    ):
-                        if columns_columnname != prefix_columnname:
-                            raise KeyError(f"{columns_columnname}")
-                else:
-                    self.columns = list(self.prefix.keys())
-                # Convert prefix from dict to list, to simplify tile work
-                self.prefix = list(self.prefix.values())
-
-        return self.new_dataframe(
-            [data],
+        if not self.columns:
+            self.agg_results = data.agg(["unique"])
+        else:
+            self.agg_results = data[self.columns].agg(["unique"])
+
+        return self.new_tileable(
+            [data, self.agg_results],
             shape=(np.nan, np.nan),
             dtypes=None,
             index_value=data.index_value,
@@ -127,7 +106,7 @@ def get_dummies(
     drop_first : bool, default False
         Whether to get k-1 dummies out of k categorical levels by removing the
         first level.
-    dtype : dtype, default np.uint8
+    dtype : dtype, default bool
         Data type for new columns. Only a single dtype is allowed.
 
     Returns
@@ -195,6 +174,56 @@ def get_dummies(
     if columns is not None and not isinstance(columns, list):
         raise TypeError("Input must be a list-like for parameter `columns`")
 
+    if isinstance(data, (list, tuple)):
+        data = asseries(data)
+    elif isinstance(data, pd.Series):
+        data = from_pandas_series(data)
+    elif isinstance(data, pd.DataFrame):
+        data = from_pandas_df(data)
+
+    dtype = dtype if dtype is not None else np.dtype(bool)
+
+    if prefix is not None:
+        if isinstance(prefix, list):
+            if columns is not None:
+                encoding_col_num = len(columns)
+            else:
+                encoding_col_num = 0
+                for dt in data.dtypes.values:
+                    if dt.kind in _encoding_dtype_kind:
+                        encoding_col_num += 1
+            prefix_num = len(prefix)
+            if prefix_num != encoding_col_num:
+                raise ValueError(
+                    f"Length of 'prefix' ({prefix_num}) did not match "
+                    + f"the length of the columns being encoded ({encoding_col_num})"
+                )
+        elif isinstance(prefix, dict):
+            if columns is not None:
+                encoding_col_num = len(columns)
+                prefix_num = len(prefix)
+                if prefix_num != encoding_col_num:
+                    raise ValueError(
+                        f"Length of 'prefix' ({prefix_num}) did not match "
+                        + f"the length of the columns being encoded ({encoding_col_num})"
+                    )
+                prefix_cols = prefix.keys()
+                for columns_columnname, prefix_columnname in zip(
+                    prefix_cols, list(columns)
+                ):
+                    if columns_columnname != prefix_columnname:
+                        raise KeyError(f"{columns_columnname}")
+            else:
+                columns = list(prefix.keys())
+            # Convert prefix from dict to list, to simplify tile work
+            prefix = list(prefix.values())
+
+    if not columns and data.ndim == 2:
+        columns = []
+        for col_name, dt in data.dtypes.items():
+            if dt.kind in _encoding_dtype_kind:
+                columns.append(col_name)
+
     op = DataFrameGetDummies(
         prefix=prefix,
         prefix_sep=prefix_sep,

maxframe/dataframe/misc/isin.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
+
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_list_like
 
 from ... import opcodes
-from ...core import ENTITY_TYPE
+from ...core import ENTITY_TYPE, EntityData
 from ...serialization.serializables import AnyField, KeyField
 from ...tensor.core import TENSOR_TYPE
 from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
@@ -30,21 +32,22 @@ class DataFrameIsin(DataFrameOperator, DataFrameOperatorMixin):
     input = KeyField("input")
     values = AnyField("values", default=None)
 
-    def _set_inputs(self, inputs):
-        super()._set_inputs(inputs)
-        inputs_iter = iter(self._inputs)
-        self.input = next(inputs_iter)
-        if len(self._inputs) > 1:
-            if isinstance(self.values, dict):
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameIsin", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        inputs_iter = iter(op._inputs)
+        op.input = next(inputs_iter)
+        if len(op._inputs) > 1:
+            if isinstance(op.values, dict):
                 new_values = dict()
-                for k, v in self.values.items():
+                for k, v in op.values.items():
                     if isinstance(v, ENTITY_TYPE):
                         new_values[k] = next(inputs_iter)
                     else:
                         new_values[k] = v
-                self.values = new_values
+                op.values = new_values
             else:
-                self.values = self._inputs[1]
+                op.values = op._inputs[1]
 
     def __call__(self, elements):
         inputs = [elements]

maxframe/dataframe/misc/map.py

@@ -13,14 +13,15 @@
 # limitations under the License.
 
 import inspect
-from collections.abc import MutableMapping
+from typing import List, MutableMapping, Union
 
 import numpy as np
 import pandas as pd
 
 from ... import opcodes
-from ...core import OutputType
+from ...core import EntityData, OutputType
 from ...serialization.serializables import AnyField, KeyField, StringField
+from ...udf import BuiltinFunction, MarkedFunction
 from ...utils import quiet_stdio
 from ..core import SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
@@ -41,11 +42,17 @@ class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
         if hasattr(self, "arg"):
             copy_func_scheduling_hints(self.arg, self)
 
-    def _set_inputs(self, inputs):
-        super()._set_inputs(inputs)
-        self.input = self._inputs[0]
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameMap", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        op.input = op._inputs[0]
         if len(inputs) == 2:
-            self.arg = self._inputs[1]
+            op.arg = op._inputs[1]
+
+    def has_custom_code(self) -> bool:
+        return not isinstance(
+            self.arg, (dict, SERIES_TYPE, pd.Series)
+        ) and not isinstance(self.arg, BuiltinFunction)
 
     def __call__(self, series, dtype, skip_infer=False):
         if dtype is None and not skip_infer:
@@ -112,6 +119,14 @@ class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
                 name=series.name,
             )
 
+    @classmethod
+    def estimate_size(
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameMap"
+    ) -> None:
+        if isinstance(op.arg, MarkedFunction):
+            ctx[op.outputs[0].key] = float("inf")
+        super().estimate_size(ctx, op)
+
 
 def series_map(
     series, arg, na_action=None, dtype=None, memory_scale=None, skip_infer=False

maxframe/dataframe/misc/melt.py

@@ -16,7 +16,7 @@ import numpy as np
 import pandas as pd
 
 from ... import opcodes
-from ...serialization.serializables import AnyField, StringField
+from ...serialization.serializables import AnyField, BoolField, StringField
 from ..operators import DataFrameOperator, DataFrameOperatorMixin, OutputType
 from ..utils import build_empty_df, parse_index
 
@@ -29,6 +29,7 @@ class DataFrameMelt(DataFrameOperator, DataFrameOperatorMixin):
     var_name = StringField("var_name", default=None)
     value_name = StringField("value_name", default=None)
     col_level = AnyField("col_level", default=None)
+    ignore_index = BoolField("ignore_index", default=False)
 
     def __call__(self, df):
         empty_result = build_empty_df(df.dtypes).melt(
@@ -37,6 +38,7 @@ class DataFrameMelt(DataFrameOperator, DataFrameOperatorMixin):
             var_name=self.var_name,
             value_name=self.value_name,
             col_level=self.col_level,
+            ignore_index=self.ignore_index,
         )
         self._output_types = [OutputType.dataframe]
         return self.new_tileable(
@@ -55,6 +57,7 @@ def melt(
     var_name=None,
     value_name="value",
     col_level=None,
+    ignore_index=False,
 ):
     """
     Unpivot a DataFrame from wide to long format, optionally leaving identifiers set.
@@ -79,6 +82,9 @@ def melt(
         Name to use for the 'value' column.
     col_level : int or str, optional
         If columns are a MultiIndex then use this level to melt.
+    ignore_index : bool, default True
+        If True, original index is ignored. If False, the original index
+        is retained. Index labels will be repeated as necessary.
 
     Returns
     -------
@@ -158,5 +164,6 @@ def melt(
         var_name=var_name,
         value_name=value_name,
         col_level=col_level,
+        ignore_index=ignore_index,
     )
     return op(frame)

maxframe/dataframe/misc/pivot.py

@@ -0,0 +1,232 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_list_like
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import AnyField, KeyField
+from ...utils import no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_df, make_column_list, parse_index
+
+
+class DataFramePivot(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.PIVOT
+
+    values = AnyField("values", default=None)
+    index = AnyField("index", default=None)
+    columns = AnyField("columns", default=None)
+
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, aggfunc=None, **kw):
+        if aggfunc is None:
+            aggfunc = "mean"
+        super().__init__(aggfunc=aggfunc, **kw)
+        self._output_types = [OutputType.dataframe]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFramePivot", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:  # pragma: no branch
+            op.agg_results = inputs[-1]
+
+    def __call__(self, df):
+        index_list = make_column_list(self.index, df.dtypes)
+        columns_list = make_column_list(self.columns, df.dtypes)
+
+        if not index_list:
+            index_data = pd.Index([])
+        elif len(index_list) == 1:
+            index_data = pd.Index(
+                [], dtype=df.dtypes[index_list[0]], name=index_list[0]
+            )
+        else:
+            index_data = pd.MultiIndex.from_frame(build_df(df[index_list]))
+        index_value = parse_index(index_data, df)
+
+        self.agg_results = df[columns_list].drop_duplicates()
+        columns_value = dtypes = None
+
+        inputs = [df]
+        if self.agg_results is not None:
+            inputs.append(self.agg_results)
+        return self.new_dataframe(
+            inputs,
+            shape=(np.nan, np.nan),
+            dtypes=dtypes,
+            columns_value=columns_value,
+            index_value=index_value,
+        )
+
+
+def pivot(data, columns, index=None, values=None):
+    """
+    Return reshaped DataFrame organized by given index / column values.
+
+    Reshape data (produce a "pivot" table) based on column values. Uses
+    unique values from specified `index` / `columns` to form axes of the
+    resulting DataFrame. This function does not support data
+    aggregation, multiple values will result in a MultiIndex in the
+    columns. See the :ref:`User Guide <reshaping>` for more on reshaping.
+
+    Parameters
+    ----------
+    index : str or object or a list of str, optional
+        Column to use to make new frame's index. If None, uses
+        existing index.
+
+    columns : str or object or a list of str
+        Column to use to make new frame's columns.
+
+    values : str, object or a list of the previous, optional
+        Column(s) to use for populating new frame's values. If not
+        specified, all remaining columns will be used and the result will
+        have hierarchically indexed columns.
+
+    Returns
+    -------
+    DataFrame
+        Returns reshaped DataFrame.
+
+    Raises
+    ------
+    ValueError:
+        When there are any `index`, `columns` combinations with multiple
+        values. `DataFrame.pivot_table` when you need to aggregate.
+
+    See Also
+    --------
+    DataFrame.pivot_table : Generalization of pivot that can handle
+        duplicate values for one index/column pair.
+    DataFrame.unstack : Pivot based on the index values instead of a
+        column.
+    wide_to_long : Wide panel to long format. Less flexible but more
+        user-friendly than melt.
+
+    Notes
+    -----
+    For finer-tuned control, see hierarchical indexing documentation along
+    with the related stack/unstack methods.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'foo': ['one', 'one', 'one', 'two', 'two',
+    ...                            'two'],
+    ...                    'bar': ['A', 'B', 'C', 'A', 'B', 'C'],
+    ...                    'baz': [1, 2, 3, 4, 5, 6],
+    ...                    'zoo': ['x', 'y', 'z', 'q', 'w', 't']})
+    >>> df.execute()
+        foo   bar  baz  zoo
+    0   one   A    1    x
+    1   one   B    2    y
+    2   one   C    3    z
+    3   two   A    4    q
+    4   two   B    5    w
+    5   two   C    6    t
+
+    >>> df.pivot(index='foo', columns='bar', values='baz').execute()
+    bar  A   B   C
+    foo
+    one  1   2   3
+    two  4   5   6
+
+    >>> df.pivot(index='foo', columns='bar', values=['baz', 'zoo']).execute()
+          baz       zoo
+    bar   A  B  C   A  B  C
+    foo
+    one   1  2  3   x  y  z
+    two   4  5  6   q  w  t
+
+    You could also assign a list of column names or a list of index names.
+
+    >>> df = md.DataFrame({
+    ...        "lev1": [1, 1, 1, 2, 2, 2],
+    ...        "lev2": [1, 1, 2, 1, 1, 2],
+    ...        "lev3": [1, 2, 1, 2, 1, 2],
+    ...        "lev4": [1, 2, 3, 4, 5, 6],
+    ...        "values": [0, 1, 2, 3, 4, 5]})
+    >>> df.execute()
+        lev1 lev2 lev3 lev4 values
+    0   1    1    1    1    0
+    1   1    1    2    2    1
+    2   1    2    1    3    2
+    3   2    1    2    4    3
+    4   2    1    1    5    4
+    5   2    2    2    6    5
+
+    >>> df.pivot(index="lev1", columns=["lev2", "lev3"], values="values").execute()
+    lev2    1         2
+    lev3    1    2    1    2
+    lev1
+    1     0.0  1.0  2.0  NaN
+    2     4.0  3.0  NaN  5.0
+
+    >>> df.pivot(index=["lev1", "lev2"], columns=["lev3"], values="values").execute()
+          lev3    1    2
+    lev1  lev2
+       1     1  0.0  1.0
+             2  2.0  NaN
+       2     1  4.0  3.0
+             2  NaN  5.0
+
+    A ValueError is raised if there are any duplicates.
+
+    >>> df = md.DataFrame({"foo": ['one', 'one', 'two', 'two'],
+    ...                    "bar": ['A', 'A', 'B', 'C'],
+    ...                    "baz": [1, 2, 3, 4]})
+    >>> df.execute()
+       foo bar  baz
+    0  one   A    1
+    1  one   A    2
+    2  two   B    3
+    3  two   C    4
+
+    Notice that the first two rows are the same for our `index`
+    and `columns` arguments.
+
+    >>> df.pivot(index='foo', columns='bar', values='baz').execute()
+    Traceback (most recent call last):
+       ...
+    ValueError: Index contains duplicate entries, cannot reshape
+    """
+    values_list = make_column_list(values, data.dtypes)
+    index_list = make_column_list(index, data.dtypes)
+    columns_list = make_column_list(columns, data.dtypes)
+
+    name_to_attr = {"values": values_list, "index": index_list, "columns": columns_list}
+    for key, val in name_to_attr.items():
+        if val is None:
+            continue
+        if not is_list_like(val):
+            raise ValueError(f"Need to specify {key} as a list-like object.")
+        non_exist_key = next((c for c in val if c not in data.dtypes.index), no_default)
+        if non_exist_key is not no_default:
+            raise ValueError(
+                f"Column {non_exist_key} specified in {key} is not a valid column."
+            )
+
+    op = DataFramePivot(
+        values=values,
+        index=index,
+        columns=columns,
+    )
+    return op(data)

maxframe/dataframe/misc/pivot_table.py

@@ -12,16 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
+
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_list_like
 
 from ... import opcodes
-from ...core import OutputType
-from ...serialization.serializables import AnyField, BoolField, StringField
+from ...core import EntityData, OutputType
+from ...serialization.serializables import AnyField, BoolField, KeyField, StringField
 from ...utils import no_default
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
-from ..utils import build_df, parse_index
+from ..utils import build_df, make_column_list, parse_index
 
 
 class DataFramePivotTable(DataFrameOperator, DataFrameOperatorMixin):
@@ -37,35 +39,53 @@ class DataFramePivotTable(DataFrameOperator, DataFrameOperatorMixin):
     margins_name = StringField("margins_name", default=None)
     sort = BoolField("sort", default=False)
 
-    def __init__(self, **kw):
-        super().__init__(**kw)
-        self.output_types = [OutputType.dataframe]
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, aggfunc=None, **kw):
+        if aggfunc is None:
+            aggfunc = "mean"
+        super().__init__(aggfunc=aggfunc, **kw)
+        self._output_types = [OutputType.dataframe]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFramePivotTable", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:  # pragma: no branch
+            op.agg_results = inputs[-1]
 
     def __call__(self, df):
-        index_value = columns_value = dtypes = None
-        if self.index is not None:
-            # index is now a required field
-            if len(self.index) == 1:
-                index_data = pd.Index(
-                    [], dtype=df.dtypes[self.index[0]], name=self.index[0]
-                )
-            else:
-                index_data = pd.MultiIndex.from_frame(build_df(df[self.index]))
-            index_value = parse_index(index_data)
-
-            if self.columns is None:  # output columns can be determined
-                sel_df = df
-                groupby_obj = sel_df.groupby(self.index)
-                if self.values:
-                    groupby_obj = groupby_obj[self.values]
-                aggregated_df = groupby_obj.agg(self.aggfunc)
-                index_value = aggregated_df.index_value
-                columns_value = aggregated_df.columns_value
-                dtypes = aggregated_df.dtypes
-            else:
-                columns_value = dtypes = None
+        index_list = make_column_list(self.index, df.dtypes)
+        columns_list = make_column_list(self.columns, df.dtypes)
+        values_list = make_column_list(self.values, df.dtypes)
+
+        if not index_list:
+            index_data = pd.Index([])
+        elif len(index_list) == 1:
+            index_data = pd.Index(
+                [], dtype=df.dtypes[index_list[0]], name=index_list[0]
+            )
+        else:
+            index_data = pd.MultiIndex.from_frame(build_df(df[index_list]))
+        index_value = parse_index(index_data, df)
+
+        if columns_list is None:  # output columns can be determined
+            sel_df = df
+            groupby_obj = sel_df.groupby(index_list)
+            if values_list:
+                groupby_obj = groupby_obj[values_list]
+            aggregated_df = groupby_obj.agg(self.aggfunc)
+            index_value = aggregated_df.index_value
+            columns_value = aggregated_df.columns_value
+            dtypes = aggregated_df.dtypes
+        else:
+            self.agg_results = df[columns_list].drop_duplicates()
+            columns_value = dtypes = None
+
+        inputs = [df]
+        if self.agg_results is not None:
+            inputs.append(self.agg_results)
         return self.new_dataframe(
-            [df],
+            inputs,
             shape=(np.nan, np.nan),
             dtypes=dtypes,
             columns_value=columns_value,
@@ -219,17 +239,9 @@ def pivot_table(
             "No group keys passed, need to specify at least one of index or columns"
         )
 
-    def make_col_list(col):
-        try:
-            if col in data.dtypes.index:
-                return [col]
-        except TypeError:
-            return col
-        return col
-
-    values_list = make_col_list(values)
-    index_list = make_col_list(index)
-    columns_list = make_col_list(columns)
+    values_list = make_column_list(values, data.dtypes)
+    index_list = make_column_list(index, data.dtypes)
+    columns_list = make_column_list(columns, data.dtypes)
 
     name_to_attr = {"values": values_list, "index": index_list, "columns": columns_list}
     for key, val in name_to_attr.items():