maxframe 1.3.0__cp310-cp310-macosx_10_9_universal2.whl → 2.0.0__cp310-cp310-macosx_10_9_universal2.whl

maxframe/learn/utils/extmath.py

@@ -0,0 +1,176 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import tensor as mt
+
+
+# Use at least float64 for the accumulating functions to avoid precision issue
+# see https://github.com/numpy/numpy/issues/9393. The float64 is also retained
+# as it is in case the float overflows
+def _safe_accumulator_op(op, x, *args, **kwargs):
+    """
+    This function provides numpy accumulator functions with a float64 dtype
+    when used on a floating point input. This prevents accumulator overflow on
+    smaller floating point dtypes.
+
+    Parameters
+    ----------
+    op : function
+        A accumulator function such as np.mean or np.sum
+    x : numpy array
+        A tensor to apply the accumulator function
+    *args : positional arguments
+        Positional arguments passed to the accumulator function after the
+        input x
+    **kwargs : keyword arguments
+        Keyword arguments passed to the accumulator function
+
+    Returns
+    -------
+    result : The output of the accumulator function passed to this function
+    """
+    if np.issubdtype(x.dtype, np.floating) and x.dtype.itemsize < 8:
+        result = op(x, *args, **kwargs, dtype=np.float64)
+    else:
+        result = op(x, *args, **kwargs)
+    return result
+
+
+def _incremental_mean_and_var(
+    X, last_mean, last_variance, last_sample_count, sample_weight=None
+):
+    """Calculate mean update and a Youngs and Cramer variance update.
+
+    If sample_weight is given, the weighted mean and variance is computed.
+
+    Update a given mean and (possibly) variance according to new data given
+    in X. last_mean is always required to compute the new mean.
+    If last_variance is None, no variance is computed and None return for
+    updated_variance.
+
+    From the paper "Algorithms for computing the sample variance: analysis and
+    recommendations", by Chan, Golub, and LeVeque.
+
+    Parameters
+    ----------
+    X : array-like of shape (n_samples, n_features)
+        Data to use for variance update.
+
+    last_mean : array-like of shape (n_features,)
+
+    last_variance : array-like of shape (n_features,)
+
+    last_sample_count : array-like of shape (n_features,)
+        The number of samples encountered until now if sample_weight is None.
+        If sample_weight is not None, this is the sum of sample_weight
+        encountered.
+
+    sample_weight : array-like of shape (n_samples,) or None
+        Sample weights. If None, compute the unweighted mean/variance.
+
+    Returns
+    -------
+    updated_mean : ndarray of shape (n_features,)
+
+    updated_variance : ndarray of shape (n_features,)
+        None if last_variance was None.
+
+    updated_sample_count : ndarray of shape (n_features,)
+
+    Notes
+    -----
+    NaNs are ignored during the algorithm.
+
+    References
+    ----------
+    T. Chan, G. Golub, R. LeVeque. Algorithms for computing the sample
+        variance: recommendations, The American Statistician, Vol. 37, No. 3,
+        pp. 242-247
+
+    Also, see the sparse implementation of this in
+    `utils.sparsefuncs.incr_mean_variance_axis` and
+    `utils.sparsefuncs_fast.incr_mean_variance_axis0`
+    """
+    # old = stats until now
+    # new = the current increment
+    # updated = the aggregated stats
+    last_sum = last_mean * last_sample_count
+    X_nan_mask = mt.isnan(X)
+    # if mt.any(X_nan_mask):
+    #     sum_op = mt.nansum
+    # else:
+    #     sum_op = mt.sum
+    sum_op = mt.nansum
+    if sample_weight is not None:
+        # equivalent to np.nansum(X * sample_weight, axis=0)
+        # safer because np.float64(X*W) != np.float64(X)*np.float64(W)
+        new_sum = _safe_accumulator_op(
+            mt.matmul, sample_weight, mt.where(X_nan_mask, 0, X)
+        )
+        new_sample_count = _safe_accumulator_op(
+            mt.sum, sample_weight[:, None] * (~X_nan_mask), axis=0
+        )
+    else:
+        new_sum = _safe_accumulator_op(sum_op, X, axis=0)
+        n_samples = X.shape[0]
+        new_sample_count = n_samples - mt.sum(X_nan_mask, axis=0)
+
+    updated_sample_count = last_sample_count + new_sample_count
+
+    updated_mean = (last_sum + new_sum) / updated_sample_count
+
+    if last_variance is None:
+        updated_variance = None
+    else:
+        T = new_sum / new_sample_count
+        temp = X - T
+        if sample_weight is not None:
+            # equivalent to np.nansum((X-T)**2 * sample_weight, axis=0)
+            # safer because np.float64(X*W) != np.float64(X)*np.float64(W)
+            correction = _safe_accumulator_op(
+                mt.matmul, sample_weight, mt.where(X_nan_mask, 0, temp)
+            )
+            temp **= 2
+            new_unnormalized_variance = _safe_accumulator_op(
+                mt.matmul, sample_weight, mt.where(X_nan_mask, 0, temp)
+            )
+        else:
+            correction = _safe_accumulator_op(sum_op, temp, axis=0)
+            temp **= 2
+            new_unnormalized_variance = _safe_accumulator_op(sum_op, temp, axis=0)
+
+        # correction term of the corrected 2 pass algorithm.
+        # See "Algorithms for computing the sample variance: analysis
+        # and recommendations", by Chan, Golub, and LeVeque.
+        new_unnormalized_variance -= correction**2 / new_sample_count
+
+        last_unnormalized_variance = last_variance * last_sample_count
+
+        with mt.errstate(divide="ignore", invalid="ignore"):
+            last_over_new_count = last_sample_count / new_sample_count
+            updated_unnormalized_variance = (
+                last_unnormalized_variance
+                + new_unnormalized_variance
+                + last_over_new_count
+                / updated_sample_count
+                * (last_sum / last_over_new_count - new_sum) ** 2
+            )
+
+        zeros = last_sample_count == 0
+        updated_unnormalized_variance[zeros] = new_unnormalized_variance[zeros]
+        updated_variance = updated_unnormalized_variance / updated_sample_count
+
+    return updated_mean, updated_variance, updated_sample_count

maxframe/learn/utils/multiclass.py

@@ -0,0 +1,292 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Sequence
+from typing import List
+
+import numpy as np
+from scipy.sparse import spmatrix
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import ENTITY_TYPE, TILEABLE_TYPE, OutputType
+from ...core.operator import Operator
+from ...serialization.serializables import AnyField, ListField
+from ...tensor.core import TENSOR_TYPE, TensorOrder
+from ...typing_ import EntityType, TileableType
+from ...udf import builtin_function
+from ..core import LearnOperatorMixin
+
+
+class UniqueLabels(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.UNIQUE_LABELS
+
+    ys = ListField("ys")
+
+    def __call__(self, ys: List[TileableType]):
+        self._output_types = [OutputType.tensor]
+        inputs = [y for y in ys if isinstance(y, TILEABLE_TYPE)]
+        return self.new_tileable(
+            inputs,
+            shape=(np.nan,),
+            dtype=mt.tensor(ys[0]).dtype,
+            order=TensorOrder.C_ORDER,
+        )
+
+
+def unique_labels(*ys):
+    """
+    Extract an ordered array of unique labels.
+
+    We don't allow:
+        - mix of multilabel and multiclass (single label) targets
+        - mix of label indicator matrix and anything else,
+          because there are no explicit labels)
+        - mix of label indicator matrices of different sizes
+        - mix of string and integer labels
+
+    At the moment, we also don't allow "multiclass-multioutput" input type.
+
+    Parameters
+    ----------
+    *ys : array-likes
+
+    Returns
+    -------
+    out : ndarray of shape (n_unique_labels,)
+        An ordered array of unique labels.
+
+    Examples
+    --------
+    >>> from maxframe.learn.utils.multiclass import unique_labels
+    >>> unique_labels([3, 5, 5, 5, 7, 7]).execute()
+    array([3, 5, 7])
+    >>> unique_labels([1, 2, 3, 4], [2, 2, 3, 4]).execute()
+    array([1, 2, 3, 4])
+    >>> unique_labels([1, 2, 10], [5, 11]).execute()
+    array([ 1,  2,  5, 10, 11])
+    """
+    if not ys:
+        raise ValueError("No argument has been passed.")
+
+    ys = list(ys)
+    op = UniqueLabels(ys=ys)
+    return op(ys)
+
+
+class IsMultilabel(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.IS_MULTILABEL
+
+    y = AnyField("y")
+
+    def __call__(self, y):
+        self._output_types = [OutputType.tensor]
+        inputs = [y] if isinstance(y, ENTITY_TYPE) else []
+        return self.new_tileable(
+            inputs, shape=(), dtype=np.dtype(bool), order=TensorOrder.C_ORDER
+        )
+
+    @classmethod
+    def _set_inputs(cls, op: "IsMultilabel", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op._inputs:
+            op.y = op._inputs[0]
+
+
+def is_multilabel(y):
+    """
+    Check if ``y`` is in a multilabel format.
+
+    Parameters
+    ----------
+    y : numpy array of shape [n_samples]
+        Target values.
+
+    Returns
+    -------
+    out : bool,
+        Return ``True``, if ``y`` is in a multilabel format, else ```False``.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> from maxframe.learn.utils.multiclass import is_multilabel
+    >>> is_multilabel([0, 1, 0, 1]).execute()
+    False
+    >>> is_multilabel([[1], [0, 2], []]).execute()
+    False
+    >>> is_multilabel(mt.array([[1, 0], [0, 0]])).execute()
+    True
+    >>> is_multilabel(mt.array([[1], [0], [0]])).execute()
+    False
+    >>> is_multilabel(mt.array([[1, 0, 0]])).execute()
+    True
+    """
+    if not isinstance(y, ENTITY_TYPE):
+        if hasattr(y, "__array__") or isinstance(y, Sequence):
+            y = np.asarray(y)
+        yt = None
+    else:
+        yt = y = mt.tensor(y)
+
+    op = IsMultilabel(y=y)
+    return op(yt)
+
+
+class TypeOfTarget(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.TYPE_OF_TARGET
+
+    y = AnyField("y")
+
+    def __call__(self, y: TileableType):
+        self._output_types = [OutputType.tensor]
+        inputs = [y] if isinstance(y, ENTITY_TYPE) else []
+        return self.new_tileable(
+            inputs, shape=(), order=TensorOrder.C_ORDER, dtype=np.dtype(object)
+        )
+
+    @classmethod
+    def _set_inputs(cls, op: "TypeOfTarget", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op._inputs:
+            op.y = op._inputs[0]
+
+
+def type_of_target(y):
+    """
+    Determine the type of data indicated by the target.
+
+    Note that this type is the most specific type that can be inferred.
+    For example:
+
+        * ``binary`` is more specific but compatible with ``multiclass``.
+        * ``multiclass`` of integers is more specific but compatible with
+          ``continuous``.
+        * ``multilabel-indicator`` is more specific but compatible with
+          ``multiclass-multioutput``.
+
+    Parameters
+    ----------
+    y : array-like
+
+    Returns
+    -------
+    target_type : string
+        One of:
+
+        * 'continuous': `y` is an array-like of floats that are not all
+          integers, and is 1d or a column vector.
+        * 'continuous-multioutput': `y` is a 2d tensor of floats that are
+          not all integers, and both dimensions are of size > 1.
+        * 'binary': `y` contains <= 2 discrete values and is 1d or a column
+          vector.
+        * 'multiclass': `y` contains more than two discrete values, is not a
+          sequence of sequences, and is 1d or a column vector.
+        * 'multiclass-multioutput': `y` is a 2d tensor that contains more
+          than two discrete values, is not a sequence of sequences, and both
+          dimensions are of size > 1.
+        * 'multilabel-indicator': `y` is a label indicator matrix, a tensor
+          of two dimensions with at least two columns, and at most 2 unique
+          values.
+        * 'unknown': `y` is array-like but none of the above, such as a 3d
+          tensor, sequence of sequences, or a tensor of non-sequence objects.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> from maxframe.learn.utils.multiclass import type_of_target
+    >>> type_of_target([0.1, 0.6]).execute()
+    'continuous'
+    >>> type_of_target([1, -1, -1, 1]).execute()
+    'binary'
+    >>> type_of_target(['a', 'b', 'a']).execute()
+    'binary'
+    >>> type_of_target([1.0, 2.0]).execute()
+    'binary'
+    >>> type_of_target([1, 0, 2]).execute()
+    'multiclass'
+    >>> type_of_target([1.0, 0.0, 3.0]).execute()
+    'multiclass'
+    >>> type_of_target(['a', 'b', 'c']).execute()
+    'multiclass'
+    >>> type_of_target(mt.array([[1, 2], [3, 1]])).execute()
+    'multiclass-multioutput'
+    >>> type_of_target([[1, 2]]).execute()
+    'multiclass-multioutput'
+    >>> type_of_target(mt.array([[1.5, 2.0], [3.0, 1.6]])).execute()
+    'continuous-multioutput'
+    >>> type_of_target(mt.array([[0, 1], [1, 1]])).execute()
+    'multilabel-indicator'
+    """
+    if isinstance(y, TENSOR_TYPE):
+        y = mt.tensor(y)
+
+    valid_types = (Sequence, spmatrix) if spmatrix is not None else (Sequence,)
+    valid = (
+        isinstance(y, valid_types)
+        or hasattr(y, "__array__")
+        or hasattr(y, "__maxframe_tensor__")
+    ) and not isinstance(y, str)
+
+    if not valid:
+        raise ValueError(f"Expected array-like (array or non-string sequence), got {y}")
+
+    sparse_pandas = type(y).__name__ in ["SparseSeries", "SparseArray"]
+    if sparse_pandas:  # pragma: no cover
+        raise ValueError("y cannot be class 'SparseSeries' or 'SparseArray'")
+
+    if isinstance(y, ENTITY_TYPE):
+        y = mt.tensor(y)
+
+    op = TypeOfTarget(y=y)
+    return op(y)
+
+
+@builtin_function
+def _check_class_target_name_and_return(t, t_type=None):
+    t_type = t_type or t
+    if t_type not in [
+        "binary",
+        "multiclass",
+        "multiclass-multioutput",
+        "multilabel-indicator",
+        "multilabel-sequences",
+    ]:
+        raise ValueError("Unknown label type: %r" % t_type)
+    return t
+
+
+def check_classification_targets(y, return_value: bool = False):
+    """
+    Ensure that target y is of a non-regression type.
+
+    Only the following target types (as defined in type_of_target) are allowed:
+        'binary', 'multiclass', 'multiclass-multioutput',
+        'multilabel-indicator', 'multilabel-sequences'
+
+    Parameters
+    ----------
+    y : array-like
+    """
+    y_type = type_of_target(y)
+
+    y_type = y_type.mf.apply_chunk(
+        _check_class_target_name_and_return, dtype=y_type.dtype
+    )
+    if not return_value:
+        return y_type
+    y = mt.array(y)
+    return y_type, y.mf.apply_chunk(
+        _check_class_target_name_and_return, args=(y_type,), **y.params
+    )

maxframe/learn/utils/shuffle.py

@@ -0,0 +1,114 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+from collections.abc import Iterable
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import ExecutableTuple, OutputType, get_output_types
+from ...core.operator import MapReduceOperator
+from ...dataframe.utils import parse_index
+from ...serialization.serializables import FieldTypes, Int64Field, TupleField
+from ...tensor.utils import check_random_state, gen_random_seeds, validate_axis
+from ...utils import tokenize
+from ..core import LearnOperatorMixin
+from . import convert_to_tensor_or_dataframe
+
+
+def _shuffle_index_value(op, index_value, chunk_index=None):
+    key = tokenize((op._values_, chunk_index, index_value.key))
+    return parse_index(pd.Index([], index_value.to_pandas().dtype), key=key)
+
+
+class LearnShuffle(MapReduceOperator, LearnOperatorMixin):
+    _op_type_ = opcodes.PERMUTATION
+
+    axes = TupleField("axes", FieldTypes.int32)
+    seeds = TupleField("seeds", FieldTypes.uint32)
+    n_samples = Int64Field("n_samples", default=None)
+
+    reduce_sizes = TupleField("reduce_sizes", FieldTypes.uint32)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @property
+    def output_limit(self):
+        if self.stage is None:
+            return len(self.output_types)
+        return 1
+
+    def _shuffle_index_value(self, index_value):
+        return _shuffle_index_value(self, index_value)
+
+    def _shuffle_dtypes(self, dtypes):
+        seed = self.seeds[self.axes.index(1)]
+        rs = np.random.RandomState(seed)
+        shuffled_dtypes = dtypes[rs.permutation(np.arange(len(dtypes)))]
+        return shuffled_dtypes
+
+    def _calc_params(self, params):
+        axes = set(self.axes)
+        for i, output_type, param in zip(itertools.count(0), self.output_types, params):
+            if output_type == OutputType.dataframe:
+                if 0 in axes:
+                    param["index_value"] = self._shuffle_index_value(
+                        param["index_value"]
+                    )
+                if 1 in axes:
+                    dtypes = param["dtypes"] = self._shuffle_dtypes(param["dtypes"])
+                    param["columns_value"] = parse_index(dtypes.index, store_data=True)
+            elif output_type == OutputType.series:
+                if 0 in axes:
+                    param["index_value"] = self._shuffle_index_value(
+                        param["index_value"]
+                    )
+            param["_position_"] = i
+        return params
+
+    def __call__(self, arrays):
+        params = self._calc_params([ar.params for ar in arrays])
+        return self.new_tileables(arrays, kws=params)
+
+
+def shuffle(*arrays, random_state=None, n_samples=None, axes=None):
+    arrays = [convert_to_tensor_or_dataframe(ar) for ar in arrays]
+    axes = axes or (0,)
+    if not isinstance(axes, Iterable):
+        axes = (axes,)
+    elif not isinstance(axes, tuple):
+        axes = tuple(axes)
+    random_state = check_random_state(random_state).to_numpy()
+    if n_samples:
+        raise TypeError(f"n_samples argument of shuffle() not supported.")
+
+    max_ndim = max(ar.ndim for ar in arrays)
+    axes = tuple(np.unique([validate_axis(max_ndim, ax) for ax in axes]).tolist())
+    seeds = gen_random_seeds(len(axes), random_state)
+
+    # verify shape
+    for ax in axes:
+        shapes = {ar.shape[ax] for ar in arrays if ax < ar.ndim}
+        if len(shapes) > 1:
+            raise ValueError(f"arrays do not have same shape on axis {ax}")
+
+    op = LearnShuffle(axes=axes, seeds=seeds, output_types=get_output_types(*arrays))
+    shuffled_arrays = op(arrays)
+    if len(arrays) == 1:
+        return shuffled_arrays[0]
+    else:
+        return ExecutableTuple(shuffled_arrays)

maxframe/learn/utils/sparsefuncs.py

@@ -0,0 +1,87 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional
+
+import numpy as np
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import OutputType
+from ...core.operator import Operator, TileableOperatorMixin
+from ...serialization.serializables import Int16Field, ReferenceField
+from ...typing_ import EntityType
+
+
+class LearnCountNonzero(Operator, TileableOperatorMixin):
+    _op_module_ = "learn"
+    _op_type_ = opcodes.COUNT_NONZERO
+
+    axis = Int16Field("axis")
+    sample_weight = ReferenceField("sample_weight")
+
+    @classmethod
+    def _set_inputs(cls, op: "LearnCountNonzero", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op.sample_weight is not None:
+            op.sample_weight = inputs[-1]
+
+    def __call__(self, x, sample_weight=None):
+        self.sample_weight = sample_weight
+        self._output_types = [
+            OutputType.scalar if self.axis is None else OutputType.tensor
+        ]
+        dtype = np.dtype(int)
+        inputs = [x]
+        if sample_weight is not None:
+            dtype = sample_weight.dtype
+            inputs = [x, sample_weight]
+
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = (x.shape[1 - self.axis],)
+
+        return self.new_tileable(inputs, shape=shape, dtype=dtype)
+
+
+def count_nonzero(X, axis: Optional[int] = None, sample_weight=None):
+    """A variant of X.getnnz() with extension to weighting on axis 0
+
+    Useful in efficiently calculating multilabel metrics.
+
+    Parameters
+    ----------
+    X : CSR sparse matrix of shape (n_samples, n_labels)
+        Input data.
+
+    axis : None, 0 or 1
+        The axis on which the data is aggregated.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Weight for each row of X.
+    """
+    if axis == -1:
+        axis = 1
+    elif axis == -2:
+        axis = 0
+    if axis is not None and axis not in (0, 1):
+        raise ValueError(f"Unsupported axis: {axis}")
+
+    X = mt.asarray(X)
+    if sample_weight is not None:
+        sample_weight = mt.asarray(sample_weight)
+
+    op = LearnCountNonzero(axis=axis)
+    return op(X, sample_weight=sample_weight)