maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/learn/preprocessing/_label/_label_encoder.py

@@ -0,0 +1,174 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from .... import tensor as mt
+from ....udf import builtin_function
+from ...core import BaseEstimator, TransformerMixin
+from ...utils._encode import _encode, _unique
+from ...utils.validation import _num_samples, column_or_1d
+
+try:
+    from sklearn.utils.validation import check_is_fitted
+except ImportError:
+    check_is_fitted = lambda *_: None
+
+
+class LabelEncoder(TransformerMixin, BaseEstimator):
+    """Encode target labels with value between 0 and n_classes-1.
+
+    This transformer should be used to encode target values, *i.e.* `y`, and
+    not the input `X`.
+
+    Read more in the :ref:`User Guide <preprocessing_targets>`.
+
+    Attributes
+    ----------
+    classes_ : ndarray of shape (n_classes,)
+        Holds the label for each class.
+
+    See Also
+    --------
+    OrdinalEncoder : Encode categorical features using an ordinal encoding
+        scheme.
+    OneHotEncoder : Encode categorical features as a one-hot numeric array.
+
+    Examples
+    --------
+    `LabelEncoder` can be used to normalize labels.
+
+    >>> from maxframe.learn import preprocessing
+    >>> le = preprocessing.LabelEncoder()
+    >>> le.fit([1, 2, 2, 6]).execute()
+    LabelEncoder()
+    >>> le.classes_.to_numpy()
+    array([1, 2, 6])
+    >>> le.transform([1, 1, 2, 6]).to_numpy()
+    array([0, 0, 1, 2]...)
+    >>> le.inverse_transform([0, 0, 1, 2]).to_numpy()
+    array([1, 1, 2, 6])
+
+    It can also be used to transform non-numerical labels (as long as they are
+    hashable and comparable) to numerical labels.
+
+    >>> le = preprocessing.LabelEncoder()
+    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"]).execute()
+    LabelEncoder()
+    >>> list(le.classes_.to_numpy())
+    ['amsterdam', 'paris', 'tokyo']
+    >>> le.transform(["tokyo", "tokyo", "paris"]).to_numpy()
+    array([2, 2, 1]...)
+    >>> list(le.inverse_transform([2, 2, 1]).to_numpy())
+    ['tokyo', 'tokyo', 'paris']
+    """
+
+    def fit(self, y, execute=False, session=None, run_kwargs=None):
+        """Fit label encoder.
+
+        Parameters
+        ----------
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        Returns
+        -------
+        self : returns an instance of self.
+            Fitted label encoder.
+        """
+        y = column_or_1d(y, warn=True)
+        self.classes_ = _unique(y)
+        if execute:
+            self.execute(session=session, **(run_kwargs or dict()))
+        return self
+
+    def fit_transform(self, y, execute=False, session=None, run_kwargs=None):
+        """Fit label encoder and return encoded labels.
+
+        Parameters
+        ----------
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        Returns
+        -------
+        y : array-like of shape (n_samples,)
+            Encoded labels.
+        """
+        y = column_or_1d(y, warn=True)
+        self.classes_, y = _unique(y, return_inverse=True)
+        if execute:
+            self.execute(session=session, extra_tileables=y, **(run_kwargs or dict()))
+        return y
+
+    def transform(self, y, execute=False, session=None, run_kwargs=None):
+        """Transform labels to normalized encoding.
+
+        Parameters
+        ----------
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        Returns
+        -------
+        y : array-like of shape (n_samples,)
+            Labels as normalized encodings.
+        """
+        check_is_fitted(self)
+        y = column_or_1d(y, warn=True)
+        # transform of empty array is empty array
+        if _num_samples(y) == 0:
+            return mt.array([])
+
+        t = _encode(y, uniques=self.classes_)
+        if execute:
+            t = t.execute(session=session, **(run_kwargs or dict()))
+        return t
+
+    @staticmethod
+    @builtin_function
+    def _class_checker(chunk_data, classes_data):
+        diff = np.setdiff1d(chunk_data, np.arange(len(classes_data)))
+        if len(diff):
+            raise ValueError("y contains previously unseen labels: %s" % str(diff))
+        return classes_data[chunk_data]
+
+    def inverse_transform(self, y, execute=False, session=None, run_kwargs=None):
+        """Transform labels back to original encoding.
+
+        Parameters
+        ----------
+        y : ndarray of shape (n_samples,)
+            Target values.
+
+        Returns
+        -------
+        y : ndarray of shape (n_samples,)
+            Original encoding.
+        """
+        check_is_fitted(self)
+        y = column_or_1d(y, warn=True)
+        # inverse transform of empty array is empty array
+        if _num_samples(y) == 0:
+            return mt.array([])
+
+        labels = mt.asarray(y).mf.apply_chunk(
+            self._class_checker, args=(self.classes_,), dtype=self.classes_.dtype
+        )
+        if execute:
+            labels.execute(session=session, **(run_kwargs or dict()))
+        return labels
+
+    def _more_tags(self):
+        return {"X_types": ["1dlabels"]}

maxframe/learn/utils/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import convert_to_tensor_or_dataframe
+from .multiclass import check_classification_targets
+from .odpsio import read_odps_model
+from .shuffle import shuffle
+from .sparsefuncs import count_nonzero
+from .validation import check_array, check_consistent_length

maxframe/learn/utils/_encode.py

@@ -0,0 +1,312 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import NamedTuple
+
+import numpy as np
+
+from maxframe import dataframe as md
+from maxframe import tensor as mt
+
+from ...udf import builtin_function
+from .core import is_scalar_nan
+
+
+@builtin_function
+def _cut_before(c, idx):
+    return c[: idx + 1]
+
+
+@builtin_function
+def _map_inverse_encodes(c, idx):
+    if c.flags.writeable:
+        c[c > idx] = idx
+    else:  # pragma: no cover
+        # If c is got from the shared memory, it is immutable.
+        c = np.select([c <= idx], [c], idx)
+    return c
+
+
+def _unique(values, *, return_inverse=False):
+    """Helper function to find unique values with support for python objects.
+
+    Uses pure python method for object dtype, and numpy method for
+    all other dtypes.
+
+    Parameters
+    ----------
+    values : ndarray
+        Values to check for unknowns.
+
+    return_inverse : bool, default=False
+        If True, also return the indices of the unique values.
+
+    Returns
+    -------
+    unique : ndarray
+        The sorted unique values.
+
+    unique_inverse : ndarray
+        The indices to reconstruct the original array from the unique array.
+        Only provided if `return_inverse` is True.
+    """
+    if values.dtype == object:
+        series_unique = md.Series(md.Series(values).unique()).sort_values().values
+        if return_inverse:
+            return series_unique, _map_to_integer(values, series_unique)
+        else:
+            return series_unique
+
+    out = mt.unique(values, return_inverse=return_inverse)
+
+    if return_inverse:
+        uniques, inverse = out
+    else:
+        uniques = out
+
+    # np.unique will have duplicate missing values at the end of `uniques`
+    # here we clip the nans and remove it from uniques
+    uniques = uniques.rechunk(tuple((s,) for s in uniques.shape))
+    nan_idx = mt.searchsorted(uniques, mt.nan)
+    uniques = uniques.mf.apply_chunk(
+        _cut_before,
+        args=(nan_idx,),
+        dtype=uniques.dtype,
+        shape=(np.nan,) * uniques.ndim,
+    )
+    if return_inverse:
+        inverse = inverse.mf.apply_chunk(
+            _map_inverse_encodes,
+            args=(nan_idx,),
+            dtype=inverse.dtype,
+            shape=((np.nan,),) * inverse.ndim,
+        )
+
+        return uniques, inverse
+    return uniques
+
+
+class _nandict(dict):  # pragma: no cover
+    """Dictionary with support for nans."""
+
+    def __init__(self, mapping):
+        super().__init__(mapping)
+        for key, value in mapping.items():
+            if is_scalar_nan(key):
+                self.nan_value = value
+                break
+
+    def __missing__(self, key):
+        if hasattr(self, "nan_value") and is_scalar_nan(key):
+            return self.nan_value
+        raise KeyError(key)
+
+
+class MissingValues(NamedTuple):  # pragma: no cover
+    """Data class for missing data information"""
+
+    nan: bool
+    none: bool
+
+    def to_list(self):
+        """Convert tuple to a list where None is always first."""
+        output = []
+        if self.none:
+            output.append(None)
+        if self.nan:
+            output.append(np.nan)
+        return output
+
+
+def _extract_missing(values):  # pragma: no cover
+    """Extract missing values from `values`.
+
+    Parameters
+    ----------
+    values: set
+        Set of values to extract missing from.
+
+    Returns
+    -------
+    output: set
+        Set with missing values extracted.
+
+    missing_values: MissingValues
+        Object with missing value information.
+    """
+    missing_values_set = {
+        value for value in values if value is None or is_scalar_nan(value)
+    }
+
+    if not missing_values_set:
+        return values, MissingValues(nan=False, none=False)
+
+    if None in missing_values_set:
+        if len(missing_values_set) == 1:
+            output_missing_values = MissingValues(nan=False, none=True)
+        else:
+            # If there is more than one missing value, then it has to be
+            # float('nan') or np.nan
+            output_missing_values = MissingValues(nan=True, none=True)
+    else:
+        output_missing_values = MissingValues(nan=True, none=False)
+
+    # create set without the missing values
+    output = values - missing_values_set
+    return output, output_missing_values
+
+
+@builtin_function
+def _value_to_int_mapper(values_data, uniques_data, check_unknown=True):
+    if values_data.dtype.kind in "OUS":
+        try:
+            table = _nandict({val: i for i, val in enumerate(uniques_data)})
+            return np.array([table[v] for v in values_data])
+        except KeyError as e:
+            raise ValueError(f"y contains previously unseen labels: {str(e)}")
+    else:
+        if check_unknown:
+            diff = _check_unknown(values_data, uniques_data)
+            if diff:
+                raise ValueError(f"y contains previously unseen labels: {str(diff)}")
+        return np.searchsorted(uniques_data, values_data)
+
+
+def _map_to_integer(values, uniques, check_unknown=True):
+    """Map values based on its position in uniques."""
+
+    return values.mf.apply_chunk(
+        _value_to_int_mapper,
+        args=(uniques,),
+        dtype=np.dtype(np.int64),
+        shape=values.shape,
+        check_unknown=check_unknown,
+    )
+
+
+def _check_unknown(values, known_values, return_mask=False):  # pragma: no cover
+    """
+    Helper function to check for unknowns in values to be encoded.
+
+    Uses pure python method for object dtype, and numpy method for
+    all other dtypes.
+
+    Parameters
+    ----------
+    values : array
+        Values to check for unknowns.
+    known_values : array
+        Known values. Must be unique.
+    return_mask : bool, default=False
+        If True, return a mask of the same shape as `values` indicating
+        the valid values.
+
+    Returns
+    -------
+    diff : list
+        The unique values present in `values` and not in `know_values`.
+    valid_mask : boolean array
+        Additionally returned if ``return_mask=True``.
+
+    """
+    valid_mask = None
+
+    if values.dtype.kind in "OUS":
+        values_set = set(values)
+        values_set, missing_in_values = _extract_missing(values_set)
+
+        uniques_set = set(known_values)
+        uniques_set, missing_in_uniques = _extract_missing(uniques_set)
+        diff = values_set - uniques_set
+
+        nan_in_diff = missing_in_values.nan and not missing_in_uniques.nan
+        none_in_diff = missing_in_values.none and not missing_in_uniques.none
+
+        def is_valid(value):
+            return (
+                value in uniques_set
+                or missing_in_uniques.none
+                and value is None
+                or missing_in_uniques.nan
+                and is_scalar_nan(value)
+            )
+
+        if return_mask:
+            if diff or nan_in_diff or none_in_diff:
+                valid_mask = np.array([is_valid(value) for value in values])
+            else:
+                valid_mask = np.ones(len(values), dtype=bool)
+
+        diff = list(diff)
+        if none_in_diff:
+            diff.append(None)
+        if nan_in_diff:
+            diff.append(np.nan)
+    else:
+        unique_values = np.unique(values)
+        diff = np.setdiff1d(unique_values, known_values, assume_unique=True)
+        if return_mask:
+            if diff.size:
+                valid_mask = np.in1d(values, known_values)
+            else:
+                valid_mask = np.ones(len(values), dtype=bool)
+
+        # check for nans in the known_values
+        if np.isnan(known_values).any():
+            diff_is_nan = np.isnan(diff)
+            if diff_is_nan.any():
+                # removes nan from valid_mask
+                if diff.size and return_mask:
+                    is_nan = np.isnan(values)
+                    valid_mask[is_nan] = 1
+
+                # remove nan from diff
+                diff = diff[~diff_is_nan]
+        diff = list(diff)
+
+    if return_mask:
+        return diff, valid_mask
+    return diff
+
+
+def _encode(values, *, uniques, check_unknown=True):
+    """Helper function to encode values into [0, n_uniques - 1].
+
+    Uses pure python method for object dtype, and numpy method for
+    all other dtypes.
+    The numpy method has the limitation that the `uniques` need to
+    be sorted. Importantly, this is not checked but assumed to already be
+    the case. The calling method needs to ensure this for all non-object
+    values.
+
+    Parameters
+    ----------
+    values : tensor
+        Values to encode.
+    uniques : tensor
+        The unique values in `values`. If the dtype is not object, then
+        `uniques` needs to be sorted.
+    check_unknown : bool, default=True
+        If True, check for values in `values` that are not in `unique`
+        and raise an error. This is ignored for object dtype, and treated as
+        True in this case. This parameter is useful for
+        _BaseEncoder._transform() to avoid calling _check_unknown()
+        twice.
+
+    Returns
+    -------
+    encoded : tensor
+        Encoded values
+    """
+    return _map_to_integer(values, uniques, check_unknown=check_unknown)

maxframe/learn/utils/checks.py

@@ -0,0 +1,160 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+
+from ... import opcodes
+from ... import tensor as mt
+from ...config import options
+from ...core import ENTITY_TYPE, EntityData, OutputType, get_output_types
+from ...core.operator import Operator
+from ...serialization.serializables import (
+    BoolField,
+    DataTypeField,
+    KeyField,
+    StringField,
+)
+from ...tensor.core import TensorOrder
+from ..core import LearnOperatorMixin
+
+try:
+    from sklearn import get_config as get_sklearn_config
+except ImportError:  # pragma: no cover
+    get_sklearn_config = None
+
+
+class CheckBase(Operator, LearnOperatorMixin):
+    input = KeyField("input")
+    value = KeyField("value", default=None)
+    err_msg = StringField("err_msg", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "CheckBase", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.input is not None:
+            op.input = op._inputs[0]
+        if op.value is not None:
+            op.value = op._inputs[-1]
+
+    def __call__(self, x, value=None):
+        # output input if value not specified
+        self.value = value = value if value is not None else x
+        self.output_types = get_output_types(value)
+        return self.new_tileable([x, value], kws=[value.params])
+
+
+class CheckNonNegative(CheckBase):
+    _op_type_ = opcodes.CHECK_NON_NEGATIVE
+
+    whom = StringField("whom", default=None)
+
+    def __init__(self, gpu=None, output_types=None, **kw):
+        super().__init__(output_types=output_types, gpu=gpu, **kw)
+        if self.err_msg is None and self.whom is not None:
+            self.err_msg = f"Negative values in data passed to {self.whom}"
+
+
+def check_non_negative_then_return_value(to_check, value, whom):
+    op = CheckNonNegative(
+        input=to_check, value=value, whom=whom, sparse=to_check.is_sparse()
+    )
+    return op(to_check, value)
+
+
+class AssertAllFinite(Operator, LearnOperatorMixin):
+    _op_type_ = opcodes.ASSERT_ALL_FINITE
+
+    x = KeyField("x")
+    allow_nan = BoolField("allow_nan", default=None)
+    msg_dtype = DataTypeField("msg_dtype", default=None)
+    check_only = BoolField("check_only", default=None)
+    input_name = StringField("input_name", default=None)
+    # chunks
+    is_finite = KeyField("is_finite", default=None)
+    check_nan = KeyField("check_nan", default=None)
+
+    def __init__(self, x=None, output_types=None, **kw):
+        super().__init__(x=x, _output_types=output_types, **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "AssertAllFinite", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        inputs_iter = iter(op.inputs)
+        for attr in ("x", "is_finite", "check_nan"):
+            if getattr(op, attr) is not None:
+                setattr(op, attr, next(inputs_iter))
+
+    @classmethod
+    def _assume_finite(cls):
+        assume_finite = options.learn.assume_finite
+        if assume_finite is None and get_sklearn_config is not None:
+            # get config from scikit-learn
+            assume_finite = get_sklearn_config()["assume_finite"]
+        if assume_finite is None:  # pragma: no cover
+            assume_finite = False
+
+        return assume_finite
+
+    def __call__(self, x):
+        if self._assume_finite():
+            # skip check
+            if self.check_only:
+                return
+            else:
+                return x
+
+        if self.check_only:
+            return self.new_tileable(
+                [x], dtype=np.dtype(bool), shape=(), order=TensorOrder.C_ORDER
+            )
+        else:
+            return self.new_tileable([x], kws=[x.params])
+
+
+def assert_all_finite(
+    X, allow_nan=False, msg_dtype=None, check_only=True, input_name=""
+):
+    if not isinstance(X, ENTITY_TYPE):
+        X = mt.asarray(X)
+
+    if (
+        isinstance(X.op, AssertAllFinite)
+        and X.op.allow_nan == allow_nan
+        and X.op.msg_dtype == msg_dtype
+        and X.op.check_only == check_only
+    ):
+        return X
+
+    if check_only:
+        output_types = [OutputType.tensor]
+        sparse = False
+    else:
+        output_types = get_output_types(X)
+        sparse = X.issparse()
+
+    op = AssertAllFinite(
+        x=X,
+        allow_nan=allow_nan,
+        msg_dtype=msg_dtype,
+        check_only=check_only,
+        sparse=sparse,
+        output_types=output_types,
+        input_name=input_name,
+    )
+    return op(X)