maxframe 2.2.0__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

maxframe/learn/utils/core.py

@@ -0,0 +1,121 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import numbers
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from ...utils import parse_readable_size
+
+try:
+    from sklearn import get_config as sklearn_get_config
+except ImportError:
+    sklearn_get_config = None
+
+from ...config import options
+from ...dataframe import DataFrame, Series
+from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from ...tensor import tensor as astensor
+
+
+def convert_to_tensor_or_dataframe(item):
+    if isinstance(item, (DATAFRAME_TYPE, pd.DataFrame)):
+        item = DataFrame(item)
+    elif isinstance(item, (SERIES_TYPE, pd.Series)):
+        item = Series(item)
+    else:
+        item = astensor(item)
+    return item
+
+
+def is_scalar_nan(x):
+    """Tests if x is NaN.
+
+    This function is meant to overcome the issue that np.isnan does not allow
+    non-numerical types as input, and that np.nan is not float('nan').
+
+    Parameters
+    ----------
+    x : any type
+
+    Returns
+    -------
+    boolean
+
+    Examples
+    --------
+    >>> is_scalar_nan(np.nan)
+    True
+    >>> is_scalar_nan(float("nan"))
+    True
+    >>> is_scalar_nan(None)
+    False
+    >>> is_scalar_nan("")
+    False
+    >>> is_scalar_nan([np.nan])
+    False
+    """
+    return isinstance(x, numbers.Real) and math.isnan(x)
+
+
+def get_chunk_n_rows(row_bytes, max_n_rows=None, working_memory=None):
+    """Calculates how many rows can be processed within working_memory
+
+    Parameters
+    ----------
+    row_bytes : int
+        The expected number of bytes of memory that will be consumed
+        during the processing of each row.
+    max_n_rows : int, optional
+        The maximum return value.
+    working_memory : int or float, optional
+        The number of rows to fit inside this number of MiB will be returned.
+        When None (default), the value of
+        ``sklearn.get_config()['working_memory']`` is used.
+
+    Returns
+    -------
+    int or the value of n_samples
+
+    Warns
+    -----
+    Issues a UserWarning if ``row_bytes`` exceeds ``working_memory`` MiB.
+    """
+
+    if working_memory is None:  # pragma: no cover
+        working_memory = options.learn.working_memory
+        if working_memory is None and sklearn_get_config is not None:
+            working_memory = sklearn_get_config()["working_memory"]
+        elif working_memory is None:
+            working_memory = 1024
+
+    if isinstance(working_memory, int):
+        working_memory *= 2**20
+    else:
+        working_memory = parse_readable_size(working_memory)[0]
+
+    chunk_n_rows = int(working_memory // row_bytes)
+    if max_n_rows is not None:
+        chunk_n_rows = min(chunk_n_rows, max_n_rows)
+    if chunk_n_rows < 1:  # pragma: no cover
+        warnings.warn(
+            "Could not adhere to working_memory config. "
+            "Currently %.0fMiB, %.0fMiB required."
+            % (working_memory, np.ceil(row_bytes * 2**-20))
+        )
+        chunk_n_rows = 1
+    return chunk_n_rows

maxframe/learn/utils/extmath.py

@@ -0,0 +1,213 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import tensor as mt
+
+
+# Use at least float64 for the accumulating functions to avoid precision issue
+# see https://github.com/numpy/numpy/issues/9393. The float64 is also retained
+# as it is in case the float overflows
+def _safe_accumulator_op(op, x, *args, **kwargs):
+    """
+    This function provides numpy accumulator functions with a float64 dtype
+    when used on a floating point input. This prevents accumulator overflow on
+    smaller floating point dtypes.
+
+    Parameters
+    ----------
+    op : function
+        A accumulator function such as np.mean or np.sum
+    x : numpy array
+        A tensor to apply the accumulator function
+    *args : positional arguments
+        Positional arguments passed to the accumulator function after the
+        input x
+    **kwargs : keyword arguments
+        Keyword arguments passed to the accumulator function
+
+    Returns
+    -------
+    result : The output of the accumulator function passed to this function
+    """
+    if np.issubdtype(x.dtype, np.floating) and x.dtype.itemsize < 8:
+        result = op(x, *args, **kwargs, dtype=np.float64)
+    else:
+        result = op(x, *args, **kwargs)
+    return result
+
+
+def logsumexp_real(a, axis=None, keepdims=False):
+    """Simplified logsumexp for real arrays without biases"""
+    from ... import tensor as mt
+
+    x = mt.tensor(a)
+    x_max = mt.amax(a, axis=axis, keepdims=True)
+    exp_x_shifted = mt.exp(x - x_max)
+    ret = mt.log(mt.sum(exp_x_shifted, axis=axis, keepdims=True)) + x_max
+    if keepdims:
+        return ret
+    return mt.squeeze(ret, axis=1)
+
+
+def _incremental_mean_and_var(
+    X, last_mean, last_variance, last_sample_count, sample_weight=None
+):
+    """Calculate mean update and a Youngs and Cramer variance update.
+
+    If sample_weight is given, the weighted mean and variance is computed.
+
+    Update a given mean and (possibly) variance according to new data given
+    in X. last_mean is always required to compute the new mean.
+    If last_variance is None, no variance is computed and None return for
+    updated_variance.
+
+    From the paper "Algorithms for computing the sample variance: analysis and
+    recommendations", by Chan, Golub, and LeVeque.
+
+    Parameters
+    ----------
+    X : array-like of shape (n_samples, n_features)
+        Data to use for variance update.
+
+    last_mean : array-like of shape (n_features,)
+
+    last_variance : array-like of shape (n_features,)
+
+    last_sample_count : array-like of shape (n_features,)
+        The number of samples encountered until now if sample_weight is None.
+        If sample_weight is not None, this is the sum of sample_weight
+        encountered.
+
+    sample_weight : array-like of shape (n_samples,) or None
+        Sample weights. If None, compute the unweighted mean/variance.
+
+    Returns
+    -------
+    updated_mean : ndarray of shape (n_features,)
+
+    updated_variance : ndarray of shape (n_features,)
+        None if last_variance was None.
+
+    updated_sample_count : ndarray of shape (n_features,)
+
+    Notes
+    -----
+    NaNs are ignored during the algorithm.
+
+    References
+    ----------
+    T. Chan, G. Golub, R. LeVeque. Algorithms for computing the sample
+        variance: recommendations, The American Statistician, Vol. 37, No. 3,
+        pp. 242-247
+
+    Also, see the sparse implementation of this in
+    `utils.sparsefuncs.incr_mean_variance_axis` and
+    `utils.sparsefuncs_fast.incr_mean_variance_axis0`
+    """
+    # old = stats until now
+    # new = the current increment
+    # updated = the aggregated stats
+    last_sum = last_mean * last_sample_count
+    X_nan_mask = mt.isnan(X)
+    # if mt.any(X_nan_mask):
+    #     sum_op = mt.nansum
+    # else:
+    #     sum_op = mt.sum
+    sum_op = mt.nansum
+    if sample_weight is not None:
+        # equivalent to np.nansum(X * sample_weight, axis=0)
+        # safer because np.float64(X*W) != np.float64(X)*np.float64(W)
+        new_sum = _safe_accumulator_op(
+            mt.matmul, sample_weight, mt.where(X_nan_mask, 0, X)
+        )
+        new_sample_count = _safe_accumulator_op(
+            mt.sum, sample_weight[:, None] * (~X_nan_mask), axis=0
+        )
+    else:
+        new_sum = _safe_accumulator_op(sum_op, X, axis=0)
+        n_samples = X.shape[0]
+        new_sample_count = n_samples - mt.sum(X_nan_mask, axis=0)
+
+    updated_sample_count = last_sample_count + new_sample_count
+
+    updated_mean = (last_sum + new_sum) / updated_sample_count
+
+    if last_variance is None:
+        updated_variance = None
+    else:
+        T = new_sum / new_sample_count
+        temp = X - T
+        if sample_weight is not None:
+            # equivalent to np.nansum((X-T)**2 * sample_weight, axis=0)
+            # safer because np.float64(X*W) != np.float64(X)*np.float64(W)
+            correction = _safe_accumulator_op(
+                mt.matmul, sample_weight, mt.where(X_nan_mask, 0, temp)
+            )
+            temp **= 2
+            new_unnormalized_variance = _safe_accumulator_op(
+                mt.matmul, sample_weight, mt.where(X_nan_mask, 0, temp)
+            )
+        else:
+            correction = _safe_accumulator_op(sum_op, temp, axis=0)
+            temp **= 2
+            new_unnormalized_variance = _safe_accumulator_op(sum_op, temp, axis=0)
+
+        # correction term of the corrected 2 pass algorithm.
+        # See "Algorithms for computing the sample variance: analysis
+        # and recommendations", by Chan, Golub, and LeVeque.
+        new_unnormalized_variance -= correction**2 / new_sample_count
+
+        last_unnormalized_variance = last_variance * last_sample_count
+
+        with mt.errstate(divide="ignore", invalid="ignore"):
+            last_over_new_count = last_sample_count / new_sample_count
+            updated_unnormalized_variance = (
+                last_unnormalized_variance
+                + new_unnormalized_variance
+                + last_over_new_count
+                / updated_sample_count
+                * (last_sum / last_over_new_count - new_sum) ** 2
+            )
+
+        zeros = last_sample_count == 0
+        updated_unnormalized_variance[zeros] = new_unnormalized_variance[zeros]
+        updated_variance = updated_unnormalized_variance / updated_sample_count
+
+    return updated_mean, updated_variance, updated_sample_count
+
+
+def row_norms(X, squared=False):
+    """Row-wise (squared) Euclidean norm of X.
+
+    Performs no input validation.
+
+    Parameters
+    ----------
+    X : array_like
+        The input tensor
+    squared : bool, optional (default = False)
+        If True, return squared norms.
+
+    Returns
+    -------
+    array_like
+        The row-wise (squared) Euclidean norm of X.
+    """
+
+    norms = (X**2).sum(axis=1)
+    if not squared:
+        norms = mt.sqrt(norms)
+    return norms

maxframe/learn/utils/multiclass.py

@@ -0,0 +1,292 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Sequence
+from typing import List
+
+import numpy as np
+from scipy.sparse import spmatrix
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import ENTITY_TYPE, TILEABLE_TYPE, OutputType
+from ...core.operator import Operator
+from ...serialization.serializables import AnyField, ListField
+from ...tensor.core import TENSOR_TYPE, TensorOrder
+from ...typing_ import EntityType, TileableType
+from ...udf import builtin_function
+from ..core import LearnOperatorMixin
+
+
+class UniqueLabels(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.UNIQUE_LABELS
+
+    ys = ListField("ys")
+
+    def __call__(self, ys: List[TileableType]):
+        self._output_types = [OutputType.tensor]
+        inputs = [y for y in ys if isinstance(y, TILEABLE_TYPE)]
+        return self.new_tileable(
+            inputs,
+            shape=(np.nan,),
+            dtype=mt.tensor(ys[0]).dtype,
+            order=TensorOrder.C_ORDER,
+        )
+
+
+def unique_labels(*ys):
+    """
+    Extract an ordered array of unique labels.
+
+    We don't allow:
+        - mix of multilabel and multiclass (single label) targets
+        - mix of label indicator matrix and anything else,
+          because there are no explicit labels)
+        - mix of label indicator matrices of different sizes
+        - mix of string and integer labels
+
+    At the moment, we also don't allow "multiclass-multioutput" input type.
+
+    Parameters
+    ----------
+    *ys : array-likes
+
+    Returns
+    -------
+    out : ndarray of shape (n_unique_labels,)
+        An ordered array of unique labels.
+
+    Examples
+    --------
+    >>> from maxframe.learn.utils.multiclass import unique_labels
+    >>> unique_labels([3, 5, 5, 5, 7, 7]).execute()
+    array([3, 5, 7])
+    >>> unique_labels([1, 2, 3, 4], [2, 2, 3, 4]).execute()
+    array([1, 2, 3, 4])
+    >>> unique_labels([1, 2, 10], [5, 11]).execute()
+    array([ 1,  2,  5, 10, 11])
+    """
+    if not ys:
+        raise ValueError("No argument has been passed.")
+
+    ys = list(ys)
+    op = UniqueLabels(ys=ys)
+    return op(ys)
+
+
+class IsMultilabel(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.IS_MULTILABEL
+
+    y = AnyField("y")
+
+    def __call__(self, y):
+        self._output_types = [OutputType.tensor]
+        inputs = [y] if isinstance(y, ENTITY_TYPE) else []
+        return self.new_tileable(
+            inputs, shape=(), dtype=np.dtype(bool), order=TensorOrder.C_ORDER
+        )
+
+    @classmethod
+    def _set_inputs(cls, op: "IsMultilabel", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op._inputs:
+            op.y = op._inputs[0]
+
+
+def is_multilabel(y):
+    """
+    Check if ``y`` is in a multilabel format.
+
+    Parameters
+    ----------
+    y : numpy array of shape [n_samples]
+        Target values.
+
+    Returns
+    -------
+    out : bool,
+        Return ``True``, if ``y`` is in a multilabel format, else ```False``.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> from maxframe.learn.utils.multiclass import is_multilabel
+    >>> is_multilabel([0, 1, 0, 1]).execute()
+    False
+    >>> is_multilabel([[1], [0, 2], []]).execute()
+    False
+    >>> is_multilabel(mt.array([[1, 0], [0, 0]])).execute()
+    True
+    >>> is_multilabel(mt.array([[1], [0], [0]])).execute()
+    False
+    >>> is_multilabel(mt.array([[1, 0, 0]])).execute()
+    True
+    """
+    if not isinstance(y, ENTITY_TYPE):
+        if hasattr(y, "__array__") or isinstance(y, Sequence):
+            y = np.asarray(y)
+        yt = None
+    else:
+        yt = y = mt.tensor(y)
+
+    op = IsMultilabel(y=y)
+    return op(yt)
+
+
+class TypeOfTarget(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.TYPE_OF_TARGET
+
+    y = AnyField("y")
+
+    def __call__(self, y: TileableType):
+        self._output_types = [OutputType.tensor]
+        inputs = [y] if isinstance(y, ENTITY_TYPE) else []
+        return self.new_tileable(
+            inputs, shape=(), order=TensorOrder.C_ORDER, dtype=np.dtype(object)
+        )
+
+    @classmethod
+    def _set_inputs(cls, op: "TypeOfTarget", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op._inputs:
+            op.y = op._inputs[0]
+
+
+def type_of_target(y):
+    """
+    Determine the type of data indicated by the target.
+
+    Note that this type is the most specific type that can be inferred.
+    For example:
+
+        * ``binary`` is more specific but compatible with ``multiclass``.
+        * ``multiclass`` of integers is more specific but compatible with
+          ``continuous``.
+        * ``multilabel-indicator`` is more specific but compatible with
+          ``multiclass-multioutput``.
+
+    Parameters
+    ----------
+    y : array-like
+
+    Returns
+    -------
+    target_type : string
+        One of:
+
+        * 'continuous': `y` is an array-like of floats that are not all
+          integers, and is 1d or a column vector.
+        * 'continuous-multioutput': `y` is a 2d tensor of floats that are
+          not all integers, and both dimensions are of size > 1.
+        * 'binary': `y` contains <= 2 discrete values and is 1d or a column
+          vector.
+        * 'multiclass': `y` contains more than two discrete values, is not a
+          sequence of sequences, and is 1d or a column vector.
+        * 'multiclass-multioutput': `y` is a 2d tensor that contains more
+          than two discrete values, is not a sequence of sequences, and both
+          dimensions are of size > 1.
+        * 'multilabel-indicator': `y` is a label indicator matrix, a tensor
+          of two dimensions with at least two columns, and at most 2 unique
+          values.
+        * 'unknown': `y` is array-like but none of the above, such as a 3d
+          tensor, sequence of sequences, or a tensor of non-sequence objects.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> from maxframe.learn.utils.multiclass import type_of_target
+    >>> type_of_target([0.1, 0.6]).execute()
+    'continuous'
+    >>> type_of_target([1, -1, -1, 1]).execute()
+    'binary'
+    >>> type_of_target(['a', 'b', 'a']).execute()
+    'binary'
+    >>> type_of_target([1.0, 2.0]).execute()
+    'binary'
+    >>> type_of_target([1, 0, 2]).execute()
+    'multiclass'
+    >>> type_of_target([1.0, 0.0, 3.0]).execute()
+    'multiclass'
+    >>> type_of_target(['a', 'b', 'c']).execute()
+    'multiclass'
+    >>> type_of_target(mt.array([[1, 2], [3, 1]])).execute()
+    'multiclass-multioutput'
+    >>> type_of_target([[1, 2]]).execute()
+    'multiclass-multioutput'
+    >>> type_of_target(mt.array([[1.5, 2.0], [3.0, 1.6]])).execute()
+    'continuous-multioutput'
+    >>> type_of_target(mt.array([[0, 1], [1, 1]])).execute()
+    'multilabel-indicator'
+    """
+    if isinstance(y, TENSOR_TYPE):
+        y = mt.tensor(y)
+
+    valid_types = (Sequence, spmatrix) if spmatrix is not None else (Sequence,)
+    valid = (
+        isinstance(y, valid_types)
+        or hasattr(y, "__array__")
+        or hasattr(y, "__maxframe_tensor__")
+    ) and not isinstance(y, str)
+
+    if not valid:
+        raise ValueError(f"Expected array-like (array or non-string sequence), got {y}")
+
+    sparse_pandas = type(y).__name__ in ["SparseSeries", "SparseArray"]
+    if sparse_pandas:  # pragma: no cover
+        raise ValueError("y cannot be class 'SparseSeries' or 'SparseArray'")
+
+    if isinstance(y, ENTITY_TYPE):
+        y = mt.tensor(y)
+
+    op = TypeOfTarget(y=y)
+    return op(y)
+
+
+@builtin_function
+def _check_class_target_name_and_return(t, t_type=None):
+    t_type = t_type or t
+    if t_type not in [
+        "binary",
+        "multiclass",
+        "multiclass-multioutput",
+        "multilabel-indicator",
+        "multilabel-sequences",
+    ]:
+        raise ValueError("Unknown label type: %r" % t_type)
+    return t
+
+
+def check_classification_targets(y, return_value: bool = False):
+    """
+    Ensure that target y is of a non-regression type.
+
+    Only the following target types (as defined in type_of_target) are allowed:
+        'binary', 'multiclass', 'multiclass-multioutput',
+        'multilabel-indicator', 'multilabel-sequences'
+
+    Parameters
+    ----------
+    y : array-like
+    """
+    y_type = type_of_target(y)
+
+    y_type = y_type.mf.apply_chunk(
+        _check_class_target_name_and_return, dtype=y_type.dtype
+    )
+    if not return_value:
+        return y_type
+    y = mt.array(y)
+    return y_type, y.mf.apply_chunk(
+        _check_class_target_name_and_return, args=(y_type,), **y.params
+    )