maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/reduction/core.py

@@ -14,7 +14,7 @@
 
 import functools
 import inspect
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 import msgpack
@@ -26,11 +26,13 @@ from ...serialization.serializables import (
     AnyField,
     BoolField,
     DataTypeField,
+    DictField,
     Int32Field,
+    Serializable,
     StringField,
 )
 from ...typing_ import TileableType
-from ...utils import get_item_if_scalar, pd_release_version, tokenize
+from ...utils import get_item_if_scalar, get_pd_option, pd_release_version, tokenize
 from ..operators import DATAFRAME_TYPE, DataFrameOperator, DataFrameOperatorMixin
 from ..utils import (
     build_df,
@@ -48,8 +50,14 @@ _level_reduction_keep_object = pd_release_version[:2] < (1, 3)
 # results in object.
 _reduce_bool_as_object = pd_release_version[:2] != (1, 2)
 
+_idx_reduction_without_numeric_only = pd_release_version[:2] < (1, 5)
+
+NamedAgg = namedtuple("NamedAgg", ["column", "aggfunc"])
+
+
+class DataFrameReduction(DataFrameOperator):
+    _legacy_name = "DataFrameReductionOperator"  # since v2.2.0
 
-class DataFrameReductionOperator(DataFrameOperator):
     axis = AnyField("axis", default=None)
     skipna = BoolField("skipna", default=True)
     level = AnyField("level", default=None)
@@ -59,8 +67,13 @@ class DataFrameReductionOperator(DataFrameOperator):
     method = StringField("method", default=None)
 
     dtype = DataTypeField("dtype", default=None)
+    combine_size = Int32Field("combine_size", default=None)
+    use_inf_as_na = BoolField("use_inf_as_na", default=None)
 
     def __init__(self, gpu=None, sparse=None, output_types=None, **kw):
+        kw["use_inf_as_na"] = kw.pop(
+            "use_inf_as_na", get_pd_option("mode.use_inf_as_na", False)
+        )
         super().__init__(gpu=gpu, sparse=sparse, _output_types=output_types, **kw)
 
     @property
@@ -78,23 +91,28 @@ class DataFrameReductionOperator(DataFrameOperator):
         return {k: v for k, v in args.items() if v is not None}
 
 
-class DataFrameCumReductionOperator(DataFrameOperator):
+# Keep for import compatibility
+DataFrameReductionOperator = DataFrameReduction
+
+
+class DataFrameCumReduction(DataFrameOperator):
+    _legacy_name = "DataFrameCumReductionOperator"  # since v2.2.0
+
     axis = AnyField("axis", default=None)
     skipna = BoolField("skipna", default=None)
 
     dtype = DataTypeField("dtype", default=None)
+    use_inf_as_na = BoolField("use_inf_as_na", default=None)
 
     def __init__(self, gpu=None, sparse=None, output_types=None, **kw):
+        kw["use_inf_as_na"] = kw.pop(
+            "use_inf_as_na", get_pd_option("mode.use_inf_as_na", False)
+        )
         super().__init__(gpu=gpu, sparse=sparse, _output_types=output_types, **kw)
 
 
-def _default_agg_fun(value, func_name=None, **kw):
-    if value.ndim == 1:
-        kw.pop("bool_only", None)
-        kw.pop("numeric_only", None)
-        return getattr(value, func_name)(**kw)
-    else:
-        return getattr(value, func_name)(**kw)
+# Keep for import compatibility
+DataFrameCumReductionOperator = DataFrameCumReduction
 
 
 @functools.lru_cache(100)
@@ -117,6 +135,8 @@ def _get_series_reduction_dtype(
         reduced = test_series.size
     elif func_name == "str_concat":
         reduced = pd.Series([test_series.str.cat()])
+    elif func_name in ("idxmin", "idxmax", "argmin", "argmax"):
+        reduced = getattr(test_series, func_name)(axis=axis, skipna=skipna)
     else:
         reduced = getattr(test_series, func_name)(
             axis=axis, skipna=skipna, numeric_only=numeric_only
@@ -135,6 +155,8 @@ def _get_df_reduction_dtype(
         reduced = getattr(test_df, func_name)(axis=axis)
     elif func_name in ("all", "any"):
         reduced = getattr(test_df, func_name)(axis=axis, bool_only=bool_only)
+    elif _idx_reduction_without_numeric_only and func_name in ("idxmin", "idxmax"):
+        reduced = getattr(test_df, func_name)(axis=axis, skipna=skipna)
     elif func_name == "str_concat":
         reduced = test_df.apply(lambda s: s.str.cat(), axis=axis)
     else:
@@ -146,6 +168,27 @@ def _get_df_reduction_dtype(
     return reduced.dtype
 
 
+class ReductionCallable(Serializable):
+    func_name = StringField("func_name")
+    kwargs = DictField("kwargs", default=None)
+
+    def __name__(self):
+        return self.func_name
+
+    def __call__(self, value):
+        kw = self.kwargs.copy()
+        if value.ndim == 1:
+            kw.pop("bool_only", None)
+            kw.pop("numeric_only", None)
+            return getattr(value, self.func_name)(**kw)
+        else:
+            return getattr(value, self.func_name)(**kw)
+
+    def __maxframe_tokenize__(self):
+        # make sure compiled functions are correctly cached
+        return type(self), self.func_name, self.kwargs
+
+
 class DataFrameReductionMixin(DataFrameOperatorMixin):
     @classmethod
     def get_reduction_callable(cls, op):
@@ -154,9 +197,7 @@ class DataFrameReductionMixin(DataFrameOperatorMixin):
             skipna=op.skipna, numeric_only=op.numeric_only, bool_only=op.bool_only
         )
         kw = {k: v for k, v in kw.items() if v is not None}
-        fun = functools.partial(_default_agg_fun, func_name=func_name, **kw)
-        fun.__name__ = func_name
-        return fun
+        return ReductionCallable(func_name=func_name, kwargs=kw)
 
     def _call_groupby_level(self, df, level):
         return df.groupby(level=level).agg(
@@ -261,10 +302,13 @@ class DataFrameReductionMixin(DataFrameOperatorMixin):
 
         if func_name == "custom_reduction":
             empty_series = build_series(series, ensure_string=True)
-            result_scalar = getattr(self, "custom_reduction").__call_agg__(empty_series)
-            if hasattr(result_scalar, "to_pandas"):  # pragma: no cover
-                result_scalar = result_scalar.to_pandas()
-            result_dtype = pd.Series(result_scalar).dtype
+            custom_reduction_obj = getattr(self, "custom_reduction")
+            result_dtype = getattr(custom_reduction_obj, "result_dtype", None)
+            if result_dtype is None:
+                result_scalar = custom_reduction_obj.__call_agg__(empty_series)
+                if hasattr(result_scalar, "to_pandas"):  # pragma: no cover
+                    result_scalar = result_scalar.to_pandas()
+                result_dtype = pd.Series(result_scalar).dtype
         else:
             result_dtype = _get_series_reduction_dtype(
                 series.dtype,
@@ -339,6 +383,10 @@ class CustomReduction:
     def __name__(self):
         return self.name
 
+    @property
+    def result_dtype(self):
+        return None
+
     def __call__(self, value):
         if isinstance(value, ENTITY_TYPE):
             from .custom_reduction import build_custom_reduction_result
@@ -426,6 +474,8 @@ _func_name_converts = dict(
     true_divide="truediv",
     floor_divide="floordiv",
     power="pow",
+    subtract="sub",
+    multiply="mul",
 )
 _func_compile_cache = dict()  # type: Dict[str, ReductionSteps]
 
@@ -442,8 +492,8 @@ _idl_primitive_types = (
 
 IN_VAR_IDL_OP = "in_var"
 OUT_VAR_IDL_OP = "out_var"
-MASK_VAR_OP = "mask"
-WHERE_VAR_OP = "where"
+MASK_VAR_IDL_OP = "mask"
+WHERE_VAR_IDL_OP = "where"
 LET_VAR_OP = "let"
 UNARY_IDL_OP_PREFIX = "unary:"
 BINARY_IDL_OP_PREFIX = "bin:"
@@ -471,7 +521,7 @@ class ReductionCompiler:
     def _check_function_valid(cls, func):
         if isinstance(func, functools.partial):
             return cls._check_function_valid(func.func)
-        elif isinstance(func, CustomReduction):
+        elif not hasattr(func, "__code__"):
             return
 
         func_code = func.__code__
@@ -569,6 +619,7 @@ class ReductionCompiler:
         from ..datasource.dataframe import DataFrameDataSource
         from ..datasource.series import SeriesDataSource
         from ..indexing.where import DataFrameWhere
+        from .custom_reduction import DataFrameCustomReduction
 
         func_token = tokenize(func, self._axis, func_name, ndim)
         if func_token in _func_compile_cache:
@@ -639,6 +690,9 @@ class ReductionCompiler:
             else:
                 map_func_name, agg_func_name = step_func_name, step_func_name
 
+            if isinstance(t.op, DataFrameCustomReduction):
+                custom_reduction = custom_reduction or t.op.custom_reduction
+
             # build agg description
             agg_funcs.append(
                 ReductionAggStep(
@@ -705,7 +759,7 @@ class ReductionCompiler:
         input_op_types = (
             DataFrameDataSource,
             SeriesDataSource,
-            DataFrameReductionOperator,
+            DataFrameReduction,
         )
 
         def _gen_expr_str(t):
@@ -776,9 +830,11 @@ class ReductionCompiler:
                     cond = _interpret_var(t.op.condition)
                     x = _interpret_var(t.op.x)
                     y = _interpret_var(t.op.y)
-                    statements = [[WHERE_VAR_OP, var_name, [cond, x, y], {}]]
+                    statements = [[WHERE_VAR_IDL_OP, var_name, [cond, x, y], {}]]
                 elif isinstance(t.op, DataFrameWhere):
-                    func_name = MASK_VAR_OP if t.op.replace_true else WHERE_VAR_OP
+                    func_name = (
+                        MASK_VAR_IDL_OP if t.op.replace_true else WHERE_VAR_IDL_OP
+                    )
                     inp = _interpret_var(t.op.input)
                     cond = _interpret_var(t.op.cond)
                     other = _interpret_var(t.op.other)

maxframe/dataframe/reduction/count.py

@@ -14,10 +14,18 @@
 
 from ... import opcodes
 from ...core import OutputType
-from .core import DataFrameReductionMixin, DataFrameReductionOperator
+from .core import DataFrameReduction, DataFrameReductionMixin, ReductionCallable
 
 
-class DataFrameCount(DataFrameReductionOperator, DataFrameReductionMixin):
+class CountReductionCallable(ReductionCallable):
+    def __call__(self, value):
+        skipna, numeric_only = self.kwargs["skipna"], self.kwargs["numeric_only"]
+        if value.ndim == 1:
+            return value.count()
+        return value.count(skipna=skipna, numeric_only=numeric_only)
+
+
+class DataFrameCount(DataFrameReduction, DataFrameReductionMixin):
     _op_type_ = opcodes.COUNT
     _func_name = "count"
 
@@ -28,13 +36,9 @@ class DataFrameCount(DataFrameReductionOperator, DataFrameReductionMixin):
     @classmethod
     def get_reduction_callable(cls, op):
         skipna, numeric_only = op.skipna, op.numeric_only
-
-        def count(value):
-            if value.ndim == 1:
-                return value.count()
-            return value.count(skipna=skipna, numeric_only=numeric_only)
-
-        return count
+        return CountReductionCallable(
+            func_name="count", kwargs={"skipna": skipna, "numeric_only": numeric_only}
+        )
 
 
 def count_series(series, level=None, **kw):

maxframe/dataframe/reduction/cov.py

@@ -0,0 +1,166 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def cov_dataframe(df, min_periods=None, ddof=1, numeric_only=True):
+    """
+    Compute pairwise covariance of columns, excluding NA/null values.
+
+    Compute the pairwise covariance among the series of a DataFrame.
+    The returned data frame is the `covariance matrix
+    <https://en.wikipedia.org/wiki/Covariance_matrix>`__ of the columns
+    of the DataFrame.
+
+    Both NA and null values are automatically excluded from the
+    calculation. (See the note below about bias from missing values.)
+    A threshold can be set for the minimum number of
+    observations for each value created. Comparisons with observations
+    below this threshold will be returned as ``NaN``.
+
+    This method is generally used for the analysis of time series data to
+    understand the relationship between different measures
+    across time.
+
+    Parameters
+    ----------
+    min_periods : int, optional
+        Minimum number of observations required per pair of columns
+        to have a valid result.
+
+    ddof : int, default 1
+        Delta degrees of freedom.  The divisor used in calculations
+        is ``N - ddof``, where ``N`` represents the number of elements.
+        This argument is applicable only when no ``nan`` is in the dataframe.
+
+    numeric_only : bool, default True
+        Include only `float`, `int` or `boolean` data.
+
+    Returns
+    -------
+    DataFrame
+        The covariance matrix of the series of the DataFrame.
+
+    See Also
+    --------
+    Series.cov : Compute covariance with another Series.
+    core.window.ewm.ExponentialMovingWindow.cov : Exponential weighted sample
+        covariance.
+    core.window.expanding.Expanding.cov : Expanding sample covariance.
+    core.window.rolling.Rolling.cov : Rolling sample covariance.
+
+    Notes
+    -----
+    Returns the covariance matrix of the DataFrame's time series.
+    The covariance is normalized by N-ddof.
+
+    For DataFrames that have Series that are missing data (assuming that
+    data is `missing at random
+    <https://en.wikipedia.org/wiki/Missing_data#Missing_at_random>`__)
+    the returned covariance matrix will be an unbiased estimate
+    of the variance and covariance between the member Series.
+
+    However, for many applications this estimate may not be acceptable
+    because the estimate covariance matrix is not guaranteed to be positive
+    semi-definite. This could lead to estimate correlations having
+    absolute values which are greater than one, and/or a non-invertible
+    covariance matrix. See `Estimation of covariance matrices
+    <https://en.wikipedia.org/w/index.php?title=Estimation_of_covariance_
+    matrices>`__ for more details.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([(1, 2), (0, 3), (2, 0), (1, 1)],
+    ...                   columns=['dogs', 'cats'])
+    >>> df.cov().execute()
+              dogs      cats
+    dogs  0.666667 -1.000000
+    cats -1.000000  1.666667
+
+    >>> mt.random.seed(42)
+    >>> df = md.DataFrame(mt.random.randn(1000, 5),
+    ...                   columns=['a', 'b', 'c', 'd', 'e'])
+    >>> df.cov().execute()
+              a         b         c         d         e
+    a  0.998438 -0.020161  0.059277 -0.008943  0.014144
+    b -0.020161  1.059352 -0.008543 -0.024738  0.009826
+    c  0.059277 -0.008543  1.010670 -0.001486 -0.000271
+    d -0.008943 -0.024738 -0.001486  0.921297 -0.013692
+    e  0.014144  0.009826 -0.000271 -0.013692  0.977795
+
+    **Minimum number of periods**
+
+    This method also supports an optional ``min_periods`` keyword
+    that specifies the required minimum number of non-NA observations for
+    each column pair in order to have a valid result:
+
+    >>> mt.random.seed(42)
+    >>> df = md.DataFrame(mt.random.randn(20, 3),
+    ...                   columns=['a', 'b', 'c'])
+    >>> df.loc[df.index[:5], 'a'] = mt.nan
+    >>> df.loc[df.index[5:10], 'b'] = mt.nan
+    >>> df.cov(min_periods=12).execute()
+              a         b         c
+    a  0.316741       NaN -0.150812
+    b       NaN  1.248003  0.191417
+    c -0.150812  0.191417  0.895202
+    """
+    from ..statistics.corr import DataFrameCorr
+
+    if not numeric_only:
+        raise NotImplementedError("numeric_only==True not supported")
+    op = DataFrameCorr(method="cov", min_periods=min_periods, ddof=ddof)
+    return op(df)
+
+
+def cov_series(series, other, min_periods=None, ddof=1):
+    """
+    Compute covariance with Series, excluding missing values.
+
+    The two `Series` objects are not required to be the same length and
+    will be aligned internally before the covariance is calculated.
+
+    Parameters
+    ----------
+    other : Series
+        Series with which to compute the covariance.
+    min_periods : int, optional
+        Minimum number of observations needed to have a valid result.
+    ddof : int, default 1
+        Delta degrees of freedom.  The divisor used in calculations
+        is ``N - ddof``, where ``N`` represents the number of elements.
+
+    Returns
+    -------
+    float
+        Covariance between Series and other normalized by N-1
+        (unbiased estimator).
+
+    See Also
+    --------
+    DataFrame.cov : Compute pairwise covariance of columns.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s1 = md.Series([0.90010907, 0.13484424, 0.62036035])
+    >>> s2 = md.Series([0.12528585, 0.26962463, 0.51111198])
+    >>> s1.cov(s2).execute()
+    -0.01685762652715874
+    """
+    from ..statistics.corr import DataFrameCorr
+
+    op = DataFrameCorr(other=other, method="cov", min_periods=min_periods, ddof=ddof)
+    return op(series)

maxframe/dataframe/reduction/cummax.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from ... import opcodes
-from .core import DataFrameCumReductionMixin, DataFrameCumReductionOperator
+from .core import DataFrameCumReduction, DataFrameCumReductionMixin
 
 
-class DataFrameCummax(DataFrameCumReductionOperator, DataFrameCumReductionMixin):
+class DataFrameCummax(DataFrameCumReduction, DataFrameCumReductionMixin):
     _op_type_ = opcodes.CUMMAX
     _func_name = "cummax"
 

maxframe/dataframe/reduction/cummin.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from ... import opcodes
-from .core import DataFrameCumReductionMixin, DataFrameCumReductionOperator
+from .core import DataFrameCumReduction, DataFrameCumReductionMixin
 
 
-class DataFrameCummin(DataFrameCumReductionOperator, DataFrameCumReductionMixin):
+class DataFrameCummin(DataFrameCumReduction, DataFrameCumReductionMixin):
     _op_type_ = opcodes.CUMMIN
     _func_name = "cummin"
 

maxframe/dataframe/reduction/cumprod.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from ... import opcodes
-from .core import DataFrameCumReductionMixin, DataFrameCumReductionOperator
+from .core import DataFrameCumReduction, DataFrameCumReductionMixin
 
 
-class DataFrameCumprod(DataFrameCumReductionOperator, DataFrameCumReductionMixin):
+class DataFrameCumprod(DataFrameCumReduction, DataFrameCumReductionMixin):
     _op_type_ = opcodes.CUMPROD
     _func_name = "cumprod"
 

maxframe/dataframe/reduction/cumsum.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from ... import opcodes
-from .core import DataFrameCumReductionMixin, DataFrameCumReductionOperator
+from .core import DataFrameCumReduction, DataFrameCumReductionMixin
 
 
-class DataFrameCumsum(DataFrameCumReductionOperator, DataFrameCumReductionMixin):
+class DataFrameCumsum(DataFrameCumReduction, DataFrameCumReductionMixin):
     _op_type_ = opcodes.CUMSUM
     _func_name = "cumsum"
 

maxframe/dataframe/reduction/custom_reduction.py

@@ -15,10 +15,10 @@
 from ... import opcodes
 from ...core import OutputType
 from ...serialization.serializables import AnyField
-from .core import DataFrameReductionMixin, DataFrameReductionOperator
+from .core import DataFrameReduction, DataFrameReductionMixin
 
 
-class DataFrameCustomReduction(DataFrameReductionOperator, DataFrameReductionMixin):
+class DataFrameCustomReduction(DataFrameReduction, DataFrameReductionMixin):
     _op_type_ = opcodes.CUSTOM_REDUCTION
     _func_name = "custom_reduction"