maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/learn/metrics/pairwise/pairwise_distances_topk.py

@@ -0,0 +1,121 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+
+from .... import opcodes
+from ....config import options
+from ....core import EntityData
+from ....core.operator import OperatorStage
+from ....serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int64Field,
+    KeyField,
+)
+from ....tensor.core import TensorOrder
+from .core import PairwiseDistances
+
+
+class PairwiseDistancesTopk(PairwiseDistances):
+    _op_type_ = opcodes.PAIRWISE_DISTANCES_TOPK
+
+    x = KeyField("x")
+    y = KeyField("y")
+    k = Int64Field("k", default=None)
+    metric = AnyField("metric", default=None)
+    metric_kwargs = DictField("metric_kwargs", default=None)
+    return_index = BoolField("return_index", default=None)
+    working_memory = AnyField("working_memory", default=None)
+
+    @property
+    def output_limit(self):
+        return 2 if self.return_index else 1
+
+    @classmethod
+    def _set_inputs(cls, op: "PairwiseDistancesTopk", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.stage != OperatorStage.agg:
+            op.x, op.y = inputs[:2]
+        else:
+            op.x = op.y = None
+
+    def __call__(self, X, Y):
+        from .pairwise import pairwise_distances
+
+        # leverage pairwise_distances for checks
+        d = pairwise_distances(X, Y, metric=self.metric, **self.metric_kwargs)
+
+        if self.k > Y.shape[0]:
+            self.k = Y.shape[0]
+
+        X, Y = d.op.inputs
+
+        shape_list = [X.shape[0]]
+        shape_list.append(min(Y.shape[0], self.k))
+        shape = tuple(shape_list)
+        kws = [
+            {
+                "shape": shape,
+                "order": TensorOrder.C_ORDER,
+                "dtype": np.dtype(np.float64),
+                "_type_": "distance",
+            },
+        ]
+        if self.return_index:
+            kws.append(
+                {
+                    "shape": shape,
+                    "order": TensorOrder.C_ORDER,
+                    "dtype": np.dtype(np.int64),
+                    "_type_": "index",
+                }
+            )
+            return self.new_tensors([X, Y], kws=kws)
+        else:
+            return self.new_tensors([X, Y], kws=kws)[0]
+
+
+def pairwise_distances_topk(
+    X,
+    Y=None,
+    k=None,
+    metric="euclidean",
+    return_index=True,
+    axis=1,
+    working_memory=None,
+    **kwds
+):
+    if k is None:  # pragma: no cover
+        raise ValueError("`k` has to be specified")
+
+    if Y is None:
+        Y = X
+    if axis == 0:
+        X, Y = Y, X
+    if working_memory is None:
+        working_memory = options.learn.working_memory
+    op = PairwiseDistancesTopk(
+        x=X,
+        y=Y,
+        k=k,
+        metric=metric,
+        metric_kwargs=kwds,
+        return_index=return_index,
+        working_memory=working_memory,
+    )
+    return op(X, Y)

maxframe/learn/metrics/pairwise/rbf_kernel.py

@@ -0,0 +1,51 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .... import tensor as mt
+from .core import PairwiseDistances
+from .euclidean import euclidean_distances
+
+
+def rbf_kernel(X, Y=None, gamma=None):
+    """
+    Compute the rbf (gaussian) kernel between X and Y::
+
+        K(x, y) = exp(-gamma ||x-y||^2)
+
+    for each pair of rows x in X and y in Y.
+
+    Read more in the :ref:`User Guide <rbf_kernel>`.
+
+    Parameters
+    ----------
+    X : tensor of shape (n_samples_X, n_features)
+
+    Y : tensor of shape (n_samples_Y, n_features)
+
+    gamma : float, default None
+        If None, defaults to 1.0 / n_features
+
+    Returns
+    -------
+    kernel_matrix : tensor of shape (n_samples_X, n_samples_Y)
+    """
+
+    X, Y = PairwiseDistances.check_pairwise_arrays(X, Y)
+    if gamma is None:
+        gamma = 1.0 / X.shape[1]
+
+    K = euclidean_distances(X, Y, squared=True)
+    K *= -gamma
+    K = mt.exp(K)
+    return K

maxframe/learn/metrics/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/metrics/tests/test_scorer.py

@@ -0,0 +1,26 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from sklearn.metrics import r2_score
+
+from .. import get_scorer
+
+
+def test_get_scorer():
+    with pytest.raises(ValueError):
+        get_scorer("unknown")
+
+    assert get_scorer("r2") is not None
+    assert get_scorer(r2_score) is not None

maxframe/learn/preprocessing/_data/min_max_scaler.py

@@ -106,10 +106,11 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
     <sphx_glr_auto_examples_preprocessing_plot_all_scaling.py>`.
     """
 
-    def __init__(self, feature_range=(0, 1), copy=True, clip=False):
+    def __init__(self, feature_range=(0, 1), copy=True, clip=False, validate=True):
         self.feature_range = feature_range
         self.copy = copy
         self.clip = clip
+        self.validate = validate
 
     def _reset(self):  # pragma: no cover
         """Reset internal data-dependent state of the scaler, if necessary.
@@ -186,13 +187,14 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
             )
 
         first_pass = not hasattr(self, "n_samples_seen_")
-        X = self._validate_data(
-            X,
-            reset=first_pass,
-            estimator=self,
-            dtype=FLOAT_DTYPES,
-            force_all_finite="allow-nan",
-        )
+        if self.validate:
+            X = self._validate_data(
+                X,
+                reset=first_pass,
+                estimator=self,
+                dtype=FLOAT_DTYPES,
+                force_all_finite="allow-nan",
+            )
 
         if isinstance(X, (DATAFRAME_TYPE, SERIES_TYPE, INDEX_TYPE)):
             data_min = X.min(axis=0)
@@ -239,13 +241,14 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
         """
         check_is_fitted(self)
 
-        X = self._validate_data(
-            X,
-            copy=self.copy,
-            dtype=FLOAT_DTYPES,
-            force_all_finite="allow-nan",
-            reset=False,
-        )
+        if self.validate:
+            X = self._validate_data(
+                X,
+                copy=self.copy,
+                dtype=FLOAT_DTYPES,
+                force_all_finite="allow-nan",
+                reset=False,
+            )
 
         X *= self.scale_
         X += self.min_
@@ -290,6 +293,7 @@ def minmax_scale(
     *,
     axis=0,
     copy=True,
+    validate=True,
     execute=False,
     session=None,
     run_kwargs=None
@@ -368,21 +372,28 @@ def minmax_scale(
     """  # noqa
     # Unlike the scaler object, this function allows 1d input.
     # If copy is required, it will be done inside the scaler object.
-    X = check_array(
-        X, copy=False, ensure_2d=False, dtype=FLOAT_DTYPES, force_all_finite="allow-nan"
-    )
-    original_ndim = X.ndim
+    if validate:
+        X = check_array(
+            X,
+            copy=False,
+            ensure_2d=False,
+            dtype=FLOAT_DTYPES,
+            force_all_finite="allow-nan",
+        )
+        original_ndim = X.ndim
 
-    if original_ndim == 1:
-        X = X.reshape(X.shape[0], 1)
+        if original_ndim == 1:
+            X = X.reshape(X.shape[0], 1)
+    else:
+        original_ndim = X.ndim
 
-    s = MinMaxScaler(feature_range=feature_range, copy=copy)
+    s = MinMaxScaler(feature_range=feature_range, copy=copy, validate=validate)
     if axis == 0:
         X = s.fit_transform(X)
     else:
         X = s.fit_transform(X.T).T
 
-    if original_ndim == 1:
+    if validate and original_ndim == 1:
         X = X.ravel()
 
     if not execute:

maxframe/learn/preprocessing/_data/standard_scaler.py

@@ -156,10 +156,11 @@ class StandardScaler(TransformerMixin, BaseEstimator):
     [[3. 3.]]
     """
 
-    def __init__(self, *, copy=True, with_mean=True, with_std=True):
+    def __init__(self, *, copy=True, with_mean=True, with_std=True, validate=True):
         self.with_mean = with_mean
         self.with_std = with_std
         self.copy = copy
+        self.validate = validate
 
     def _reset(self):
         """Reset internal data-dependent state of the scaler, if necessary.
@@ -246,14 +247,15 @@ class StandardScaler(TransformerMixin, BaseEstimator):
             Fitted scaler.
         """
         first_call = not hasattr(self, "n_samples_seen_")
-        X = self._validate_data(
-            X,
-            accept_sparse=("csr", "csc"),
-            dtype=FLOAT_DTYPES,
-            force_all_finite="allow-nan",
-            reset=first_call,
-        )
-        n_features = X.shape[1]
+        if self.validate:
+            X = self._validate_data(
+                X,
+                accept_sparse=("csr", "csc"),
+                dtype=FLOAT_DTYPES,
+                force_all_finite="allow-nan",
+                reset=first_call,
+            )
+        n_features = X.shape[1] if X.ndim == 2 else 1
 
         if sample_weight is not None:
             sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
@@ -267,7 +269,9 @@ class StandardScaler(TransformerMixin, BaseEstimator):
         # incr_mean_variance_axis and _incremental_variance_axis
         dtype = np.int64 if sample_weight is None else X.dtype
         if not hasattr(self, "n_samples_seen_"):
-            self.n_samples_seen_ = mt.zeros(n_features, dtype=dtype)
+            self.n_samples_seen_ = (
+                mt.zeros(n_features, dtype=dtype) if X.ndim == 2 else 0
+            )
         # elif np.size(self.n_samples_seen_) == 1:
         #     self.n_samples_seen_ = np.repeat(self.n_samples_seen_, X.shape[1])
         #     self.n_samples_seen_ = self.n_samples_seen_.astype(dtype, copy=False)
@@ -309,9 +313,11 @@ class StandardScaler(TransformerMixin, BaseEstimator):
             constant_mask = _is_constant_feature(
                 self.var_, self.mean_, self.n_samples_seen_
             )
-            self.scale_ = _handle_zeros_in_scale(
-                mt.sqrt(self.var_), copy=False, constant_mask=constant_mask
-            )
+            self.scale_ = mt.sqrt(self.var_)
+            if self.validate:
+                self.scale_ = _handle_zeros_in_scale(
+                    self.scale_, copy=False, constant_mask=constant_mask
+                )
         else:
             self.scale_ = None
 
@@ -337,14 +343,15 @@ class StandardScaler(TransformerMixin, BaseEstimator):
         check_is_fitted(self)
 
         copy = copy if copy is not None else self.copy
-        X = self._validate_data(
-            X,
-            reset=False,
-            accept_sparse="csr",
-            copy=copy,
-            dtype=FLOAT_DTYPES,
-            force_all_finite="allow-nan",
-        )
+        if self.validate:
+            X = self._validate_data(
+                X,
+                reset=False,
+                accept_sparse="csr",
+                copy=copy,
+                dtype=FLOAT_DTYPES,
+                force_all_finite="allow-nan",
+            )
 
         if sparse.issparse(X):
             raise NotImplementedError("Scaling on sparse tensors is not supported")
@@ -397,7 +404,7 @@ class StandardScaler(TransformerMixin, BaseEstimator):
         return X
 
 
-def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
+def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True, validate=True):
     """Standardize a dataset along any axis.
 
     Center to the mean and component wise scale to unit variance.
@@ -488,16 +495,18 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
         X = mt.tensor(X)
 
     ndim = X.ndim
-    if ndim == 1:
+    if validate and ndim == 1:
         X = X.reshape((X.shape[0], 1))
     if axis == 1:
         X = X.T
 
-    scaler = StandardScaler(with_mean=with_mean, with_std=with_std, copy=copy)
+    scaler = StandardScaler(
+        with_mean=with_mean, with_std=with_std, copy=copy, validate=validate
+    )
     transformed = scaler.fit_transform(X)
 
     if axis == 1:
         transformed = transformed.T
-    if ndim == 1:
+    if validate and ndim == 1:
         transformed = transformed.reshape(transformed.shape[0])
     return transformed

maxframe/learn/utils/__init__.py

@@ -14,6 +14,7 @@
 
 from .core import convert_to_tensor_or_dataframe
 from .multiclass import check_classification_targets
+from .odpsio import read_odps_model
 from .shuffle import shuffle
 from .sparsefuncs import count_nonzero
-from .validation import check_consistent_length
+from .validation import check_array, check_consistent_length

maxframe/learn/utils/checks.py

@@ -20,7 +20,7 @@ from ... import opcodes
 from ... import tensor as mt
 from ...config import options
 from ...core import ENTITY_TYPE, EntityData, OutputType, get_output_types
-from ...core.operator import Operator, OperatorStage
+from ...core.operator import Operator
 from ...serialization.serializables import (
     BoolField,
     DataTypeField,
@@ -56,7 +56,6 @@ class CheckBase(Operator, LearnOperatorMixin):
         # output input if value not specified
         self.value = value = value if value is not None else x
         self.output_types = get_output_types(value)
-        self.stage = OperatorStage.agg
         return self.new_tileable([x, value], kws=[value.params])
 
 

maxframe/learn/utils/core.py

@@ -14,9 +14,19 @@
 
 import math
 import numbers
+import warnings
 
+import numpy as np
 import pandas as pd
 
+from ...utils import parse_readable_size
+
+try:
+    from sklearn import get_config as sklearn_get_config
+except ImportError:
+    sklearn_get_config = None
+
+from ...config import options
 from ...dataframe import DataFrame, Series
 from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
 from ...tensor import tensor as astensor
@@ -60,3 +70,52 @@ def is_scalar_nan(x):
     False
     """
     return isinstance(x, numbers.Real) and math.isnan(x)
+
+
+def get_chunk_n_rows(row_bytes, max_n_rows=None, working_memory=None):
+    """Calculates how many rows can be processed within working_memory
+
+    Parameters
+    ----------
+    row_bytes : int
+        The expected number of bytes of memory that will be consumed
+        during the processing of each row.
+    max_n_rows : int, optional
+        The maximum return value.
+    working_memory : int or float, optional
+        The number of rows to fit inside this number of MiB will be returned.
+        When None (default), the value of
+        ``sklearn.get_config()['working_memory']`` is used.
+
+    Returns
+    -------
+    int or the value of n_samples
+
+    Warns
+    -----
+    Issues a UserWarning if ``row_bytes`` exceeds ``working_memory`` MiB.
+    """
+
+    if working_memory is None:  # pragma: no cover
+        working_memory = options.learn.working_memory
+        if working_memory is None and sklearn_get_config is not None:
+            working_memory = sklearn_get_config()["working_memory"]
+        elif working_memory is None:
+            working_memory = 1024
+
+    if isinstance(working_memory, int):
+        working_memory *= 2**20
+    else:
+        working_memory = parse_readable_size(working_memory)[0]
+
+    chunk_n_rows = int(working_memory // row_bytes)
+    if max_n_rows is not None:
+        chunk_n_rows = min(chunk_n_rows, max_n_rows)
+    if chunk_n_rows < 1:  # pragma: no cover
+        warnings.warn(
+            "Could not adhere to working_memory config. "
+            "Currently %.0fMiB, %.0fMiB required."
+            % (working_memory, np.ceil(row_bytes * 2**-20))
+        )
+        chunk_n_rows = 1
+    return chunk_n_rows

maxframe/learn/utils/extmath.py

@@ -15,6 +15,9 @@
 import numpy as np
 
 from ... import tensor as mt
+from ...core import ENTITY_TYPE
+from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from ...tensor.datasource import TensorZeros
 
 
 # Use at least float64 for the accumulating functions to avoid precision issue
@@ -42,13 +45,30 @@ def _safe_accumulator_op(op, x, *args, **kwargs):
     -------
     result : The output of the accumulator function passed to this function
     """
-    if np.issubdtype(x.dtype, np.floating) and x.dtype.itemsize < 8:
+    if (
+        hasattr(x, "dtype")
+        and np.issubdtype(x.dtype, np.floating)
+        and x.dtype.itemsize < 8
+    ):
         result = op(x, *args, **kwargs, dtype=np.float64)
     else:
         result = op(x, *args, **kwargs)
     return result
 
 
+def logsumexp_real(a, axis=None, keepdims=False):
+    """Simplified logsumexp for real arrays without biases"""
+    from ... import tensor as mt
+
+    x = mt.tensor(a)
+    x_max = mt.amax(a, axis=axis, keepdims=True)
+    exp_x_shifted = mt.exp(x - x_max)
+    ret = mt.log(mt.sum(exp_x_shifted, axis=axis, keepdims=True)) + x_max
+    if keepdims:
+        return ret
+    return mt.squeeze(ret, axis=1)
+
+
 def _incremental_mean_and_var(
     X, last_mean, last_variance, last_sample_count, sample_weight=None
 ):
@@ -104,16 +124,31 @@ def _incremental_mean_and_var(
     `utils.sparsefuncs.incr_mean_variance_axis` and
     `utils.sparsefuncs_fast.incr_mean_variance_axis0`
     """
+    has_last_sample = isinstance(last_sample_count, ENTITY_TYPE) and not isinstance(
+        last_sample_count.op, TensorZeros
+    )
+    is_df_type = isinstance(X, (DATAFRAME_TYPE, SERIES_TYPE))
+
     # old = stats until now
     # new = the current increment
     # updated = the aggregated stats
-    last_sum = last_mean * last_sample_count
+    last_sum = last_mean * last_sample_count if has_last_sample else 0
     X_nan_mask = mt.isnan(X)
     # if mt.any(X_nan_mask):
     #     sum_op = mt.nansum
     # else:
     #     sum_op = mt.sum
-    sum_op = mt.nansum
+
+    def df_sum(val, **kw):
+        if "dtype" in kw:
+            val = val.astype(kw.pop("dtype"))
+        return val.sum(**kw)
+
+    if is_df_type:
+        sum_op = df_sum
+    else:
+        sum_op = mt.nansum
+
     if sample_weight is not None:
         # equivalent to np.nansum(X * sample_weight, axis=0)
         # safer because np.float64(X*W) != np.float64(X)*np.float64(W)
@@ -125,10 +160,16 @@ def _incremental_mean_and_var(
         )
     else:
         new_sum = _safe_accumulator_op(sum_op, X, axis=0)
-        n_samples = X.shape[0]
-        new_sample_count = n_samples - mt.sum(X_nan_mask, axis=0)
+        if is_df_type:
+            new_sample_count = X.count()
+        else:
+            n_samples = X.shape[0]
+            new_sample_count = n_samples - mt.sum(X_nan_mask, axis=0)
 
-    updated_sample_count = last_sample_count + new_sample_count
+    if not has_last_sample:
+        updated_sample_count = new_sample_count
+    else:
+        updated_sample_count = last_sample_count + new_sample_count
 
     updated_mean = (last_sum + new_sum) / updated_sample_count
 
@@ -157,7 +198,9 @@ def _incremental_mean_and_var(
         # and recommendations", by Chan, Golub, and LeVeque.
         new_unnormalized_variance -= correction**2 / new_sample_count
 
-        last_unnormalized_variance = last_variance * last_sample_count
+        last_unnormalized_variance = (
+            last_variance * last_sample_count if has_last_sample else 0
+        )
 
         with mt.errstate(divide="ignore", invalid="ignore"):
             last_over_new_count = last_sample_count / new_sample_count
@@ -169,8 +212,35 @@ def _incremental_mean_and_var(
                 * (last_sum / last_over_new_count - new_sum) ** 2
             )
 
-        zeros = last_sample_count == 0
-        updated_unnormalized_variance[zeros] = new_unnormalized_variance[zeros]
+        if not has_last_sample:
+            updated_unnormalized_variance = new_unnormalized_variance
+        else:
+            zeros = last_sample_count == 0
+            updated_unnormalized_variance[zeros] = new_unnormalized_variance[zeros]
         updated_variance = updated_unnormalized_variance / updated_sample_count
 
     return updated_mean, updated_variance, updated_sample_count
+
+
+def row_norms(X, squared=False):
+    """Row-wise (squared) Euclidean norm of X.
+
+    Performs no input validation.
+
+    Parameters
+    ----------
+    X : array_like
+        The input tensor
+    squared : bool, optional (default = False)
+        If True, return squared norms.
+
+    Returns
+    -------
+    array_like
+        The row-wise (squared) Euclidean norm of X.
+    """
+
+    norms = (X**2).sum(axis=1)
+    if not squared:
+        norms = mt.sqrt(norms)
+    return norms