maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/dataframe/misc/map.py

@@ -0,0 +1,236 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+from collections.abc import MutableMapping
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import AnyField, KeyField, StringField
+from ...utils import quiet_stdio
+from ..core import SERIES_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_series
+
+
+class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.MAP
+
+    input = KeyField("input", default=None)
+    arg = AnyField("arg", default=None)
+    na_action = StringField("na_action", default=None)
+
+    def __init__(self, output_types=None, memory_scale=None, **kw):
+        super().__init__(_output_types=output_types, _memory_scale=memory_scale, **kw)
+        if not self.output_types:
+            self.output_types = [OutputType.series]
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        self.input = self._inputs[0]
+        if len(inputs) == 2:
+            self.arg = self._inputs[1]
+
+    def __call__(self, series, dtype, skip_infer=False):
+        if dtype is None and not skip_infer:
+            inferred_dtype = None
+            if callable(self.arg):
+                # arg is a function, try to inspect the signature
+                sig = inspect.signature(self.arg)
+                return_type = sig.return_annotation
+                if return_type is not inspect._empty:
+                    inferred_dtype = np.dtype(return_type)
+                else:
+                    try:
+                        with quiet_stdio():
+                            # try to infer dtype by calling the function
+                            inferred_dtype = (
+                                build_series(series)
+                                .map(self.arg, na_action=self.na_action)
+                                .dtype
+                            )
+                    except:  # noqa: E722  # nosec
+                        pass
+            else:
+                if isinstance(self.arg, MutableMapping):
+                    inferred_dtype = pd.Series(self.arg).dtype
+                else:
+                    inferred_dtype = self.arg.dtype
+            if inferred_dtype is not None and np.issubdtype(inferred_dtype, np.number):
+                if np.issubdtype(inferred_dtype, np.inexact):
+                    # for the inexact e.g. float
+                    # we can make the decision,
+                    # but for int, due to the nan which may occur,
+                    # we cannot infer the dtype
+                    dtype = inferred_dtype
+            else:
+                dtype = inferred_dtype
+
+        if dtype is None:
+            if not skip_infer:
+                raise ValueError(
+                    "cannot infer dtype, it needs to be specified manually for `map`"
+                )
+        else:
+            dtype = np.int64 if dtype is int else dtype
+            dtype = np.dtype(dtype)
+
+        inputs = [series]
+        if isinstance(self.arg, SERIES_TYPE):
+            inputs.append(self.arg)
+
+        if isinstance(series, SERIES_TYPE):
+            return self.new_series(
+                inputs,
+                shape=series.shape,
+                dtype=dtype,
+                index_value=series.index_value,
+                name=series.name,
+            )
+        else:
+            return self.new_index(
+                inputs,
+                shape=series.shape,
+                dtype=dtype,
+                index_value=series.index_value,
+                name=series.name,
+            )
+
+
+def series_map(
+    series, arg, na_action=None, dtype=None, memory_scale=None, skip_infer=False
+):
+    """
+    Map values of Series according to input correspondence.
+
+    Used for substituting each value in a Series with another value,
+    that may be derived from a function, a ``dict`` or
+    a :class:`Series`.
+
+    Parameters
+    ----------
+    arg : function, collections.abc.Mapping subclass or Series
+        Mapping correspondence.
+    na_action : {None, 'ignore'}, default None
+        If 'ignore', propagate NaN values, without passing them to the
+        mapping correspondence.
+    dtype : np.dtype, default None
+        Specify return type of the function. Must be specified when
+        we cannot decide the return type of the function.
+    memory_scale : float
+        Specify the scale of memory uses in the function versus
+        input size.
+    skip_infer: bool, default False
+        Whether infer dtypes when dtypes or output_type is not specified
+
+    Returns
+    -------
+    Series
+        Same index as caller.
+
+    See Also
+    --------
+    Series.apply : For applying more complex functions on a Series.
+    DataFrame.apply : Apply a function row-/column-wise.
+    DataFrame.applymap : Apply a function elementwise on a whole DataFrame.
+
+    Notes
+    -----
+    When ``arg`` is a dictionary, values in Series that are not in the
+    dictionary (as keys) are converted to ``NaN``. However, if the
+    dictionary is a ``dict`` subclass that defines ``__missing__`` (i.e.
+    provides a method for default values), then this default is used
+    rather than ``NaN``.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(['cat', 'dog', mt.nan, 'rabbit'])
+    >>> s.execute()
+    0      cat
+    1      dog
+    2      NaN
+    3   rabbit
+    dtype: object
+
+    ``map`` accepts a ``dict`` or a ``Series``. Values that are not found
+    in the ``dict`` are converted to ``NaN``, unless the dict has a default
+    value (e.g. ``defaultdict``):
+
+    >>> s.map({'cat': 'kitten', 'dog': 'puppy'}).execute()
+    0   kitten
+    1    puppy
+    2      NaN
+    3      NaN
+    dtype: object
+
+    It also accepts a function:
+
+    >>> s.map('I am a {}'.format).execute()
+    0       I am a cat
+    1       I am a dog
+    2       I am a nan
+    3    I am a rabbit
+    dtype: object
+
+    To avoid applying the function to missing values (and keep them as
+    ``NaN``) ``na_action='ignore'`` can be used:
+
+    >>> s.map('I am a {}'.format, na_action='ignore').execute()
+    0     I am a cat
+    1     I am a dog
+    2            NaN
+    3  I am a rabbit
+    dtype: object
+    """
+    op = DataFrameMap(arg=arg, na_action=na_action, memory_scale=memory_scale)
+    return op(series, dtype=dtype, skip_infer=skip_infer)
+
+
+def index_map(
+    idx, mapper, na_action=None, dtype=None, memory_scale=None, skip_infer=False
+):
+    """
+    Map values using input correspondence (a dict, Series, or function).
+
+    Parameters
+    ----------
+    mapper : function, dict, or Series
+        Mapping correspondence.
+    na_action : {None, 'ignore'}
+        If 'ignore', propagate NA values, without passing them to the
+        mapping correspondence.
+    dtype : np.dtype, default None
+        Specify return type of the function. Must be specified when
+        we cannot decide the return type of the function.
+    memory_scale : float
+        Specify the scale of memory uses in the function versus
+        input size.
+    skip_infer: bool, default False
+        Whether infer dtypes when dtypes or output_type is not specified
+
+
+    Returns
+    -------
+    applied : Union[Index, MultiIndex], inferred
+        The output of the mapping function applied to the index.
+        If the function returns a tuple with more than one element
+        a MultiIndex will be returned.
+    """
+    op = DataFrameMap(arg=mapper, na_action=na_action, memory_scale=memory_scale)
+    return op(idx, dtype=dtype, skip_infer=skip_infer)

maxframe/dataframe/misc/melt.py

@@ -0,0 +1,162 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, StringField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin, OutputType
+from ..utils import build_empty_df, parse_index
+
+
+class DataFrameMelt(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.MELT
+
+    id_vars = AnyField("id_vars", default=None)
+    value_vars = AnyField("value_vars", default=None)
+    var_name = StringField("var_name", default=None)
+    value_name = StringField("value_name", default=None)
+    col_level = AnyField("col_level", default=None)
+
+    def __call__(self, df):
+        empty_result = build_empty_df(df.dtypes).melt(
+            id_vars=self.id_vars,
+            value_vars=self.value_vars,
+            var_name=self.var_name,
+            value_name=self.value_name,
+            col_level=self.col_level,
+        )
+        self._output_types = [OutputType.dataframe]
+        return self.new_tileable(
+            [df],
+            shape=(np.nan, len(empty_result.columns)),
+            dtypes=empty_result.dtypes,
+            index_value=parse_index(pd.RangeIndex(-1), df.key, df.index_value.key),
+            columns_value=parse_index(empty_result.columns, store_data=True),
+        )
+
+
+def melt(
+    frame,
+    id_vars=None,
+    value_vars=None,
+    var_name=None,
+    value_name="value",
+    col_level=None,
+):
+    """
+    Unpivot a DataFrame from wide to long format, optionally leaving identifiers set.
+
+    This function is useful to massage a DataFrame into a format where one
+    or more columns are identifier variables (`id_vars`), while all other
+    columns, considered measured variables (`value_vars`), are "unpivoted" to
+    the row axis, leaving just two non-identifier columns, 'variable' and
+    'value'.
+
+    Parameters
+    ----------
+    id_vars : tuple, list, or ndarray, optional
+        Column(s) to use as identifier variables.
+    value_vars : tuple, list, or ndarray, optional
+        Column(s) to unpivot. If not specified, uses all columns that
+        are not set as `id_vars`.
+    var_name : scalar
+        Name to use for the 'variable' column. If None it uses
+        ``frame.columns.name`` or 'variable'.
+    value_name : scalar, default 'value'
+        Name to use for the 'value' column.
+    col_level : int or str, optional
+        If columns are a MultiIndex then use this level to melt.
+
+    Returns
+    -------
+    DataFrame
+        Unpivoted DataFrame.
+
+    See Also
+    --------
+    melt
+    pivot_table
+    DataFrame.pivot
+    Series.explode
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'A': {0: 'a', 1: 'b', 2: 'c'},
+    ...                    'B': {0: 1, 1: 3, 2: 5},
+    ...                    'C': {0: 2, 1: 4, 2: 6}})
+    >>> df.execute()
+       A  B  C
+    0  a  1  2
+    1  b  3  4
+    2  c  5  6
+
+    >>> df.melt(id_vars=['A'], value_vars=['B']).execute()
+       A variable  value
+    0  a        B      1
+    1  b        B      3
+    2  c        B      5
+
+    >>> df.melt(id_vars=['A'], value_vars=['B', 'C']).execute()
+       A variable  value
+    0  a        B      1
+    1  b        B      3
+    2  c        B      5
+    3  a        C      2
+    4  b        C      4
+    5  c        C      6
+
+    The names of 'variable' and 'value' columns can be customized:
+
+    >>> df.melt(id_vars=['A'], value_vars=['B'],
+    ...         var_name='myVarname', value_name='myValname').execute()
+       A myVarname  myValname
+    0  a         B          1
+    1  b         B          3
+    2  c         B          5
+
+    If you have multi-index columns:
+
+    >>> df = md.DataFrame({('A', 'D'): {0: 'a', 1: 'b', 2: 'c'},
+    ...                    ('B', 'E'): {0: 1, 1: 3, 2: 5},
+    ...                    ('C', 'F'): {0: 2, 1: 4, 2: 6}})
+    >>> df.execute()
+       A  B  C
+       D  E  F
+    0  a  1  2
+    1  b  3  4
+    2  c  5  6
+
+    >>> df.melt(col_level=0, id_vars=['A'], value_vars=['B']).execute()
+       A variable  value
+    0  a        B      1
+    1  b        B      3
+    2  c        B      5
+
+    >>> df.melt(id_vars=[('A', 'D')], value_vars=[('B', 'E')]).execute()
+      (A, D) variable_0 variable_1  value
+    0      a          B          E      1
+    1      b          B          E      3
+    2      c          B          E      5
+    """
+    op = DataFrameMelt(
+        id_vars=id_vars,
+        value_vars=value_vars,
+        var_name=var_name,
+        value_name=value_name,
+        col_level=col_level,
+    )
+    return op(frame)

maxframe/dataframe/misc/memory_usage.py

@@ -0,0 +1,248 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import BoolField, Int64Field
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index
+
+
+class DataFrameMemoryUsage(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.MEMORY_USAGE
+
+    # raw arguments of memory_usage method
+    index = BoolField("index", default=None)
+    deep = BoolField("deep", default=None)
+
+    # size of range index, when set, the value will be prepended to the result series
+    # if the input is a dataframe, or added to the result when the input is a series
+    range_index_size = Int64Field("range_index_size", default=None)
+
+    def _adapt_index(self, input_index, index=0):
+        """
+        When ``index=True`` is passed, an extra column will be prepended to the result series
+        Thus we need to update the index of initial chunk for returned dataframe chunks
+        """
+        if not self.index or index != 0:
+            return input_index
+        idx_data = input_index.to_pandas().insert(0, "Index")
+        return parse_index(idx_data, store_data=True)
+
+    def _adapt_nsplits(self, input_nsplit):
+        """
+        When ``index=True`` is passed, the size of returned series is one element larger
+        than the number of columns, which affects ``nsplits``.
+        """
+        if not self.index:
+            return (input_nsplit[-1],)
+        nsplits_list = list(input_nsplit[-1])
+        nsplits_list[0] += 1
+        return (tuple(nsplits_list),)
+
+    def __call__(self, df_or_series):
+        """
+        Return output object of memory_usage() call
+        """
+        if df_or_series.ndim == 1:
+            # the input data is a series, a Scalar will be returned
+            return self.new_scalar([df_or_series], dtype=np.dtype(np.int_))
+        else:
+            # the input data is a DataFrame, a Scalar will be returned
+            # calculate shape of returning series given ``op.index``
+            new_shape = (
+                (df_or_series.shape[-1] + 1,)
+                if self.index
+                else (df_or_series.shape[-1],)
+            )
+            return self.new_series(
+                [df_or_series],
+                index_value=self._adapt_index(df_or_series.columns_value),
+                shape=new_shape,
+                dtype=np.dtype(np.int_),
+            )
+
+
+def df_memory_usage(df, index=True, deep=False):
+    """
+    Return the memory usage of each column in bytes.
+
+    The memory usage can optionally include the contribution of
+    the index and elements of `object` dtype.
+
+    This value is displayed in `DataFrame.info` by default. This can be
+    suppressed by setting ``pandas.options.display.memory_usage`` to False.
+
+    Parameters
+    ----------
+    index : bool, default True
+        Specifies whether to include the memory usage of the DataFrame's
+        index in returned Series. If ``index=True``, the memory usage of
+        the index is the first item in the output.
+    deep : bool, default False
+        If True, introspect the data deeply by interrogating
+        `object` dtypes for system-level memory consumption, and include
+        it in the returned values.
+
+    Returns
+    -------
+    Series
+        A Series whose index is the original column names and whose values
+        is the memory usage of each column in bytes.
+
+    See Also
+    --------
+    numpy.ndarray.nbytes : Total bytes consumed by the elements of an
+        ndarray.
+    Series.memory_usage : Bytes consumed by a Series.
+    Categorical : Memory-efficient array for string values with
+        many repeated values.
+    DataFrame.info : Concise summary of a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> dtypes = ['int64', 'float64', 'complex128', 'object', 'bool']
+    >>> data = dict([(t, mt.ones(shape=5000).astype(t))
+    ...              for t in dtypes])
+    >>> df = md.DataFrame(data)
+    >>> df.head().execute()
+       int64  float64            complex128  object  bool
+    0      1      1.0    1.000000+0.000000j       1  True
+    1      1      1.0    1.000000+0.000000j       1  True
+    2      1      1.0    1.000000+0.000000j       1  True
+    3      1      1.0    1.000000+0.000000j       1  True
+    4      1      1.0    1.000000+0.000000j       1  True
+
+    >>> df.memory_usage().execute()
+    Index           128
+    int64         40000
+    float64       40000
+    complex128    80000
+    object        40000
+    bool           5000
+    dtype: int64
+
+    >>> df.memory_usage(index=False).execute()
+    int64         40000
+    float64       40000
+    complex128    80000
+    object        40000
+    bool           5000
+    dtype: int64
+
+    The memory footprint of `object` dtype columns is ignored by default:
+
+    >>> df.memory_usage(deep=True).execute()
+    Index            128
+    int64          40000
+    float64        40000
+    complex128     80000
+    object        160000
+    bool            5000
+    dtype: int64
+
+    Use a Categorical for efficient storage of an object-dtype column with
+    many repeated values.
+
+    >>> df['object'].astype('category').memory_usage(deep=True).execute()
+    5216
+    """
+    op = DataFrameMemoryUsage(index=index, deep=deep)
+    return op(df)
+
+
+def series_memory_usage(series, index=True, deep=False):
+    """
+    Return the memory usage of the Series.
+
+    The memory usage can optionally include the contribution of
+    the index and of elements of `object` dtype.
+
+    Parameters
+    ----------
+    index : bool, default True
+        Specifies whether to include the memory usage of the Series index.
+    deep : bool, default False
+        If True, introspect the data deeply by interrogating
+        `object` dtypes for system-level memory consumption, and include
+        it in the returned value.
+
+    Returns
+    -------
+    int
+        Bytes of memory consumed.
+
+    See Also
+    --------
+    numpy.ndarray.nbytes : Total bytes consumed by the elements of the
+        array.
+    DataFrame.memory_usage : Bytes consumed by a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(range(3))
+    >>> s.memory_usage().execute()
+    152
+
+    Not including the index gives the size of the rest of the data, which
+    is necessarily smaller:
+
+    >>> s.memory_usage(index=False).execute()
+    24
+
+    The memory footprint of `object` values is ignored by default:
+
+    >>> s = md.Series(["a", "b"])
+    >>> s.values.execute()
+    array(['a', 'b'], dtype=object)
+
+    >>> s.memory_usage().execute()
+    144
+
+    >>> s.memory_usage(deep=True).execute()
+    260
+    """
+    op = DataFrameMemoryUsage(index=index, deep=deep)
+    return op(series)
+
+
+def index_memory_usage(index, deep=False):
+    """
+    Memory usage of the values.
+
+    Parameters
+    ----------
+    deep : bool
+        Introspect the data deeply, interrogate
+        `object` dtypes for system-level memory consumption.
+
+    Returns
+    -------
+    bytes used
+
+    See Also
+    --------
+    numpy.ndarray.nbytes
+
+    Notes
+    -----
+    Memory usage does not include memory consumed by elements that
+    are not components of the array if deep=False
+    """
+    op = DataFrameMemoryUsage(index=False, deep=deep)
+    return op(index)