maxframe 2.0.0b2__cp39-cp39-macosx_10_9_universal2.whl → 2.2.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/dataframe/misc/describe.py

@@ -22,7 +22,7 @@ from ...core import EntityData
 from ...serialization.serializables import AnyField, FieldTypes, KeyField, ListField
 from ..core import SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
-from ..utils import build_empty_df, parse_index
+from ..utils import build_df, parse_index
 
 
 class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
@@ -43,8 +43,6 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
 
     def __call__(self, df_or_series):
         if isinstance(df_or_series, SERIES_TYPE):
-            if not np.issubdtype(df_or_series.dtype, np.number):
-                raise NotImplementedError("non-numeric type is not supported for now")
             test_series = pd.Series([], dtype=df_or_series.dtype).describe(
                 percentiles=self.percentiles,
                 include=self.include,
@@ -57,7 +55,7 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
                 index_value=parse_index(test_series.index, store_data=True),
             )
         else:
-            test_inp_df = build_empty_df(df_or_series.dtypes)
+            test_inp_df = build_df(df_or_series)
             test_df = test_inp_df.describe(
                 percentiles=self.percentiles,
                 include=self.include,
@@ -69,11 +67,6 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
                 # MaxFrame DataFrame allows user to specify percentiles=False
                 # to skip computation about percentiles
                 test_df.drop(["50%"], axis=0, inplace=True)
-            for dtype in test_df.dtypes:
-                if not np.issubdtype(dtype, np.number):
-                    raise NotImplementedError(
-                        "non-numeric type is not supported for now"
-                    )
             return self.new_dataframe(
                 [df_or_series],
                 shape=test_df.shape,
@@ -84,6 +77,179 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def describe(df_or_series, percentiles=None, include=None, exclude=None):
+    """
+    Generate descriptive statistics.
+
+    Descriptive statistics include those that summarize the central
+    tendency, dispersion and shape of a
+    dataset's distribution, excluding ``NaN`` values.
+
+    Analyzes both numeric and object series, as well
+    as ``DataFrame`` column sets of mixed data types. The output
+    will vary depending on what is provided. Refer to the notes
+    below for more detail.
+
+    Parameters
+    ----------
+    percentiles : list-like of numbers, optional
+        The percentiles to include in the output. All should
+        fall between 0 and 1. The default is
+        ``[.25, .5, .75]``, which returns the 25th, 50th, and
+        75th percentiles.
+    include : 'all', list-like of dtypes or None (default), optional
+        A white list of data types to include in the result. Ignored
+        for ``Series``. Here are the options:
+
+        - 'all' : All columns of the input will be included in the output.
+        - A list-like of dtypes : Limits the results to the
+          provided data types.
+          To limit the result to numeric types submit
+          ``numpy.number``. To limit it instead to object columns submit
+          the ``numpy.object`` data type. Strings
+          can also be used in the style of
+          ``select_dtypes`` (e.g. ``df.describe(include=['O'])``).
+        - None (default) : The result will include all numeric columns.
+    exclude : list-like of dtypes or None (default), optional,
+        A black list of data types to omit from the result. Ignored
+        for ``Series``. Here are the options:
+
+        - A list-like of dtypes : Excludes the provided data types
+          from the result. To exclude numeric types submit
+          ``numpy.number``. To exclude object columns submit the data
+          type ``numpy.object``. Strings can also be used in the style of
+          ``select_dtypes`` (e.g. ``df.describe(exclude=['O'])``).
+        - None (default) : The result will exclude nothing.
+
+    Returns
+    -------
+    Series or DataFrame
+        Summary statistics of the Series or Dataframe provided.
+
+    See Also
+    --------
+    DataFrame.count: Count number of non-NA/null observations.
+    DataFrame.max: Maximum of the values in the object.
+    DataFrame.min: Minimum of the values in the object.
+    DataFrame.mean: Mean of the values.
+    DataFrame.std: Standard deviation of the observations.
+    DataFrame.select_dtypes: Subset of a DataFrame including/excluding
+        columns based on their dtype.
+
+    Notes
+    -----
+    For numeric data, the result's index will include ``count``,
+    ``mean``, ``std``, ``min``, ``max`` as well as lower, ``50`` and
+    upper percentiles. By default the lower percentile is ``25`` and the
+    upper percentile is ``75``. The ``50`` percentile is the
+    same as the median.
+
+    For object data (e.g. strings or timestamps), the result's index
+    will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``
+    is the most common value. The ``freq`` is the most common value's
+    frequency. Timestamps also include the ``first`` and ``last`` items.
+
+    If multiple object values have the highest count, then the
+    ``count`` and ``top`` results will be arbitrarily chosen from
+    among those with the highest count.
+
+    For mixed data types provided via a ``DataFrame``, the default is to
+    return only an analysis of numeric columns. If the dataframe consists
+    only of object data without any numeric columns, the default is to
+    return an analysis of object columns. If ``include='all'`` is provided
+    as an option, the result will include a union of attributes of each type.
+
+    The `include` and `exclude` parameters can be used to limit
+    which columns in a ``DataFrame`` are analyzed for the output.
+    The parameters are ignored when analyzing a ``Series``.
+
+    Examples
+    --------
+    Describing a numeric ``Series``.
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([1, 2, 3])
+    >>> s.describe().execute()
+    count    3.0
+    mean     2.0
+    std      1.0
+    min      1.0
+    25%      1.5
+    50%      2.0
+    75%      2.5
+    max      3.0
+    dtype: float64
+
+    Describing a ``DataFrame``. By default only numeric fields
+    are returned.
+
+    >>> df = md.DataFrame({'numeric': [1, 2, 3],
+    ...                    'object': ['a', 'b', 'c']
+    ...                    })
+    >>> df.describe().execute()
+           numeric
+    count      3.0
+    mean       2.0
+    std        1.0
+    min        1.0
+    25%        1.5
+    50%        2.0
+    75%        2.5
+    max        3.0
+
+    Describing all columns of a ``DataFrame`` regardless of data type.
+
+    >>> df.describe(include='all').execute()  # doctest: +SKIP.execute()
+           numeric object
+    count      3.0      3
+    unique     NaN      3
+    top        NaN      a
+    freq       NaN      1
+    mean       2.0    NaN
+    std        1.0    NaN
+    min        1.0    NaN
+    25%        1.5    NaN
+    50%        2.0    NaN
+    75%        2.5    NaN
+    max        3.0    NaN
+
+    Describing a column from a ``DataFrame`` by accessing it as
+    an attribute.
+
+    >>> df.numeric.describe().execute()
+    count    3.0
+    mean     2.0
+    std      1.0
+    min      1.0
+    25%      1.5
+    50%      2.0
+    75%      2.5
+    max      3.0
+    Name: numeric, dtype: float64
+
+    Including only numeric columns in a ``DataFrame`` description.
+
+    >>> df.describe(include=[mt.number]).execute()
+           numeric
+    count      3.0
+    mean       2.0
+    std        1.0
+    min        1.0
+    25%        1.5
+    50%        2.0
+    75%        2.5
+    max        3.0
+
+    Including only string columns in a ``DataFrame`` description.
+
+    >>> df.describe(include=[object]).execute()  # doctest: +SKIP.execute()
+           object
+    count       3
+    unique      3
+    top         a
+    freq        1
+    """
+    # fixme add support for categorical columns once implemented
     if percentiles is False:
         percentiles = []
     elif percentiles is None:

maxframe/dataframe/misc/drop_duplicates.py

@@ -19,10 +19,10 @@ from ... import opcodes
 from ...serialization.serializables import BoolField
 from ..operators import OutputType
 from ..utils import gen_unknown_index_value, parse_index
-from ._duplicate import DuplicateOperand, validate_subset
+from ._duplicate import BaseDuplicateOp, validate_subset
 
 
-class DataFrameDropDuplicates(DuplicateOperand):
+class DataFrameDropDuplicates(BaseDuplicateOp):
     _op_type_ = opcodes.DROP_DUPLICATES
 
     ignore_index = BoolField("ignore_index", default=True)

maxframe/dataframe/misc/duplicated.py

@@ -16,10 +16,10 @@ import numpy as np
 
 from ... import opcodes
 from ...core import OutputType
-from ._duplicate import DuplicateOperand, validate_subset
+from ._duplicate import BaseDuplicateOp, validate_subset
 
 
-class DataFrameDuplicated(DuplicateOperand):
+class DataFrameDuplicated(BaseDuplicateOp):
     _op_type_ = opcodes.DUPLICATED
 
     def __init__(self, output_types=None, **kw):

maxframe/dataframe/misc/get_dummies.py

@@ -25,12 +25,14 @@ from ...serialization.serializables import (
     ListField,
     StringField,
 )
+from ...utils import make_dtype, pd_release_version
 from ..datasource.dataframe import from_pandas as from_pandas_df
 from ..datasource.series import from_pandas as from_pandas_series
 from ..initializer import Series as asseries
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 
 _encoding_dtype_kind = ["O", "S", "U"]
+_ret_uint8 = pd_release_version < (2, 0, 0)
 
 
 class DataFrameGetDummies(DataFrameOperator, DataFrameOperatorMixin):
@@ -181,7 +183,9 @@ def get_dummies(
     elif isinstance(data, pd.DataFrame):
         data = from_pandas_df(data)
 
-    dtype = dtype if dtype is not None else np.dtype(bool)
+    dtype = make_dtype(
+        dtype if dtype is not None else np.dtype(np.uint8 if _ret_uint8 else bool)
+    )
 
     if prefix is not None:
         if isinstance(prefix, list):

maxframe/dataframe/misc/isin.py

@@ -133,7 +133,7 @@ def series_isin(elements, values):
     5    False
     Name: animal, dtype: bool
     """
-    if is_list_like(values):
+    if is_list_like(values) and not isinstance(values, ENTITY_TYPE):
         values = list(values)
     elif not isinstance(values, (SERIES_TYPE, TENSOR_TYPE, INDEX_TYPE)):
         raise TypeError(
@@ -207,7 +207,7 @@ def df_isin(df, values):
     falcon      True       True
     dog        False      False
     """
-    if is_list_like(values) and not isinstance(values, dict):
+    if is_list_like(values) and not isinstance(values, (dict, ENTITY_TYPE)):
         values = list(values)
     elif not isinstance(
         values, (SERIES_TYPE, DATAFRAME_TYPE, TENSOR_TYPE, INDEX_TYPE, dict)

maxframe/dataframe/misc/map.py

@@ -251,3 +251,97 @@ def index_map(
     """
     op = DataFrameMap(arg=mapper, na_action=na_action, memory_scale=memory_scale)
     return op(idx, dtype=dtype, skip_infer=skip_infer)
+
+
+def df_map(
+    df, func, na_action=None, dtypes=None, dtype=None, skip_infer=False, **kwargs
+):
+    """
+    Apply a function to a Dataframe elementwise.
+
+    This method applies a function that accepts and returns a scalar
+    to every element of a DataFrame.
+
+    Parameters
+    ----------
+    func : callable
+        Python function, returns a single value from a single value.
+    na_action : {None, 'ignore'}, default None
+        If 'ignore', propagate NaN values, without passing them to func.
+    dtypes : Series, default None
+        Specify dtypes of returned DataFrames.
+    dtype : np.dtype, default None
+        Specify dtypes of all columns of returned DataFrames, only
+        effective when dtypes is not specified.
+    skip_infer: bool, default False
+        Whether infer dtypes when dtypes or dtype is not specified.
+    **kwargs
+        Additional keyword arguments to pass as keywords arguments to
+        `func`.
+
+    Returns
+    -------
+    DataFrame
+        Transformed DataFrame.
+
+    See Also
+    --------
+    DataFrame.apply : Apply a function along input axis of DataFrame.
+    DataFrame.replace: Replace values given in `to_replace` with `value`.
+    Series.map : Apply a function elementwise on a Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2.12], [3.356, 4.567]])
+    >>> df.execute()
+           0      1
+    0  1.000  2.120
+    1  3.356  4.567
+
+    >>> df.map(lambda x: len(str(x))).execute()
+       0  1
+    0  3  4
+    1  5  5
+
+    Like Series.map, NA values can be ignored:
+
+    >>> df_copy = df.copy()
+    >>> df_copy.iloc[0, 0] = md.NA
+    >>> df_copy.map(lambda x: len(str(x)), na_action='ignore').execute()
+         0  1
+    0  NaN  4
+    1  5.0  5
+
+    It is also possible to use `map` with functions that are not
+    `lambda` functions:
+
+    >>> df.map(round, ndigits=1).execute()
+         0    1
+    0  1.0  2.1
+    1  3.4  4.6
+
+    Note that a vectorized version of `func` often exists, which will
+    be much faster. You could square each number elementwise.
+
+    >>> df.map(lambda x: x**2).execute()
+               0          1
+    0   1.000000   4.494400
+    1  11.262736  20.857489
+
+    But it's better to avoid map in that case.
+
+    >>> (df ** 2).execute()
+               0          1
+    0   1.000000   4.494400
+    1  11.262736  20.857489
+    """
+    if dtypes is None and dtype is not None:
+        dtypes = pd.Series([dtype] * df.shape[1], index=df.dtypes.index)
+
+    def _wrapper(row):
+        return row.map(func, na_action=na_action, **kwargs)
+
+    return df.apply(
+        _wrapper, axis=1, dtypes=dtypes, skip_infer=skip_infer, elementwise=True
+    )

maxframe/dataframe/misc/tests/test_misc.py

@@ -16,6 +16,8 @@ import numpy as np
 import pandas as pd
 import pytest
 
+from maxframe import options
+
 from .... import opcodes
 from ....core import OutputType
 from ....dataframe import DataFrame
@@ -124,6 +126,7 @@ def test_dataframe_apply():
         dtypes=pd.Series([np.dtype(float)] * 3),
     )
     assert df2.ndim == 2
+    assert df2.op.expect_resources == options.function.default_running_options
 
 
 def test_series_apply():
@@ -180,6 +183,8 @@ def test_series_apply():
         pd.Series, output_type="dataframe", dtypes=dtypes, index=pd.RangeIndex(2)
     )
     assert r.ndim == 2
+    assert r.op.expect_resources == options.function.default_running_options
+
     pd.testing.assert_series_equal(r.dtypes, dtypes)
     assert r.shape == (2, 3)
 
@@ -305,6 +310,7 @@ def test_transform():
     assert r.shape == series.shape
     assert r.op._op_type_ == opcodes.TRANSFORM
     assert r.op.output_types[0] == OutputType.series
+    assert r.op.expect_resources == options.function.default_running_options
 
 
 def test_series_isin():
@@ -563,12 +569,17 @@ def test_apply():
     )
     assert apply_df.shape == (3, 2)
     assert apply_df.op.expect_engine == "SPE"
-    assert apply_df.op.expect_resources == {"cpu": 1, "memory": "40GB", "gpu": 0}
+    assert apply_df.op.expect_resources == {
+        "cpu": 4,
+        "memory": "40GB",
+        "gpu": 0,
+        "gu_quota": None,
+    }
 
 
 def test_pivot_table():
     from ...groupby.aggregation import DataFrameGroupByAgg
-    from ...misc.pivot_table import DataFramePivotTable
+    from ...reshape.pivot_table import DataFramePivotTable
 
     raw = pd.DataFrame(
         {

maxframe/dataframe/misc/to_numeric.py

@@ -15,6 +15,7 @@
 import numpy as np
 import pandas as pd
 
+from ... import opcodes
 from ...core import ENTITY_TYPE, OutputType
 from ...serialization.serializables import StringField
 from ...tensor import tensor as astensor
@@ -23,6 +24,8 @@ from ..operators import DataFrameOperator, DataFrameOperatorMixin
 
 
 class DataFrameToNumeric(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.TO_NUMERIC
+
     errors = StringField("errors")
     downcast = StringField("downcast")
 

maxframe/dataframe/misc/transform.py

@@ -38,8 +38,9 @@ from ..utils import (
 _with_convert_dtype = pd_release_version < (1, 2, 0)
 
 
-class TransformOperator(DataFrameOperator, DataFrameOperatorMixin):
+class DataFrameTransform(DataFrameOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.TRANSFORM
+    _legacy_name = "TransformOperator"
 
     func = AnyField("func", default=None)
     axis = AnyField("axis", default=None)
@@ -141,13 +142,17 @@ class TransformOperator(DataFrameOperator, DataFrameOperatorMixin):
 
     @classmethod
     def estimate_size(
-        cls, ctx: MutableMapping[str, Union[int, float]], op: "TransformOperator"
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameTransform"
     ) -> None:
         if isinstance(op.func, MarkedFunction):
             ctx[op.outputs[0].key] = float("inf")
         super().estimate_size(ctx, op)
 
 
+# keep for import compatibility
+TransformOperator = DataFrameTransform
+
+
 def get_packed_funcs(df, output_type, func, *args, **kwds) -> Any:
     stub_df = _build_stub_pandas_obj(df, output_type)
     n_args = copy_if_possible(args)
@@ -235,7 +240,7 @@ def df_transform(df, func, axis=0, *args, dtypes=None, skip_infer=False, **kwarg
     """
     call_agg = kwargs.pop("_call_agg", False)
     func = get_packed_funcs(df, OutputType.dataframe, func, *args, **kwargs)
-    op = TransformOperator(
+    op = DataFrameTransform(
         func=func,
         axis=axis,
         args=args,
@@ -327,13 +332,15 @@ def series_transform(
     """
     call_agg = kwargs.pop("_call_agg", False)
     func = get_packed_funcs(series, OutputType.series, func, *args, **kwargs)
-    op = TransformOperator(
+    op = DataFrameTransform(
         func=func,
         axis=axis,
         convert_dtype=convert_dtype,
         args=args,
         kwds=kwargs,
-        output_types=[OutputType.series],
+        output_types=[OutputType.series]
+        if not call_agg and not isinstance(func, list)
+        else None,
         call_agg=call_agg,
     )
     return op(series, dtype=dtype, name=series.name, skip_infer=skip_infer)

maxframe/dataframe/misc/transpose.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pandas as pd
+
 from ... import opcodes
 from ...core import OutputType
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
@@ -30,10 +32,20 @@ class DataFrameTranspose(DataFrameOperator, DataFrameOperatorMixin):
         new_shape = arg.shape[::-1]
         columns_value = arg.index_value
         index_value = parse_index(arg.dtypes.index)
+
+        if not arg.index_value.has_value:
+            dtypes = None
+        else:
+            from pandas.core.dtypes.cast import find_common_type
+
+            dtype = find_common_type(list(arg.dtypes))
+            pd_index = arg.index_value.to_pandas()
+            dtypes = pd.Series([dtype] * len(pd_index), index=pd_index)
+
         return self.new_dataframe(
             [arg],
             shape=new_shape,
-            dtypes=None,
+            dtypes=dtypes,
             columns_value=columns_value,
             index_value=index_value,
         )

maxframe/dataframe/misc/valid_index.py

@@ -0,0 +1,115 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...udf import builtin_function
+
+
+@builtin_function
+def _item_or_none(item):
+    if len(item) > 0:
+        return item[0]
+    return None
+
+
+def _valid_index(df_or_series, slc: slice):
+    from ... import tensor as mt
+
+    idx = df_or_series.dropna(how="all").index[slc]
+    return mt.array(idx).mf.apply_chunk(_item_or_none, dtype=idx.dtype)
+
+
+_doc = """
+Return index for %(pos)s non-NA value or None, if no non-NA value is found.
+
+Returns
+-------
+type of index
+
+Examples
+--------
+For Series:
+
+>>> import maxframe.dataframe as md
+>>> s = md.Series([None, 3, 4])
+>>> s.first_valid_index().execute()
+1
+>>> s.last_valid_index().execute()
+2
+
+>>> s = md.Series([None, None])
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If all elements in Series are NA/null, returns None.
+
+>>> s = md.Series()
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If Series is empty, returns None.
+
+For DataFrame:
+
+>>> df = md.DataFrame({'A': [None, None, 2], 'B': [None, 3, 4]})
+>>> df.execute()
+     A      B
+0  NaN    NaN
+1  NaN    3.0
+2  2.0    4.0
+>>> df.first_valid_index().execute()
+1
+>>> df.last_valid_index().execute()
+2
+
+>>> df = md.DataFrame({'A': [None, None, None], 'B': [None, None, None]})
+>>> df.execute()
+     A      B
+0  None   None
+1  None   None
+2  None   None
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If all elements in DataFrame are NA/null, returns None.
+
+>>> df = md.DataFrame()
+>>> df.execute()
+Empty DataFrame
+Columns: []
+Index: []
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If DataFrame is empty, returns None.
+"""
+
+
+def first_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(None, 1))
+
+
+def last_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(-1, None))
+
+
+first_valid_index.__doc__ = _doc % dict(pos="first")
+last_valid_index.__doc__ = _doc % dict(pos="last")