maxframe 2.0.0b2__cp310-cp310-win_amd64.whl → 2.2.0__cp310-cp310-win_amd64.whl

maxframe/learn/metrics/pairwise/pairwise_distances_topk.py

@@ -0,0 +1,121 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+
+from .... import opcodes
+from ....config import options
+from ....core import EntityData
+from ....core.operator import OperatorStage
+from ....serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int64Field,
+    KeyField,
+)
+from ....tensor.core import TensorOrder
+from .core import PairwiseDistances
+
+
+class PairwiseDistancesTopk(PairwiseDistances):
+    _op_type_ = opcodes.PAIRWISE_DISTANCES_TOPK
+
+    x = KeyField("x")
+    y = KeyField("y")
+    k = Int64Field("k", default=None)
+    metric = AnyField("metric", default=None)
+    metric_kwargs = DictField("metric_kwargs", default=None)
+    return_index = BoolField("return_index", default=None)
+    working_memory = AnyField("working_memory", default=None)
+
+    @property
+    def output_limit(self):
+        return 2 if self.return_index else 1
+
+    @classmethod
+    def _set_inputs(cls, op: "PairwiseDistancesTopk", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.stage != OperatorStage.agg:
+            op.x, op.y = inputs[:2]
+        else:
+            op.x = op.y = None
+
+    def __call__(self, X, Y):
+        from .pairwise import pairwise_distances
+
+        # leverage pairwise_distances for checks
+        d = pairwise_distances(X, Y, metric=self.metric, **self.metric_kwargs)
+
+        if self.k > Y.shape[0]:
+            self.k = Y.shape[0]
+
+        X, Y = d.op.inputs
+
+        shape_list = [X.shape[0]]
+        shape_list.append(min(Y.shape[0], self.k))
+        shape = tuple(shape_list)
+        kws = [
+            {
+                "shape": shape,
+                "order": TensorOrder.C_ORDER,
+                "dtype": np.dtype(np.float64),
+                "_type_": "distance",
+            },
+        ]
+        if self.return_index:
+            kws.append(
+                {
+                    "shape": shape,
+                    "order": TensorOrder.C_ORDER,
+                    "dtype": np.dtype(np.int64),
+                    "_type_": "index",
+                }
+            )
+            return self.new_tensors([X, Y], kws=kws)
+        else:
+            return self.new_tensors([X, Y], kws=kws)[0]
+
+
+def pairwise_distances_topk(
+    X,
+    Y=None,
+    k=None,
+    metric="euclidean",
+    return_index=True,
+    axis=1,
+    working_memory=None,
+    **kwds
+):
+    if k is None:  # pragma: no cover
+        raise ValueError("`k` has to be specified")
+
+    if Y is None:
+        Y = X
+    if axis == 0:
+        X, Y = Y, X
+    if working_memory is None:
+        working_memory = options.learn.working_memory
+    op = PairwiseDistancesTopk(
+        x=X,
+        y=Y,
+        k=k,
+        metric=metric,
+        metric_kwargs=kwds,
+        return_index=return_index,
+        working_memory=working_memory,
+    )
+    return op(X, Y)

maxframe/learn/metrics/pairwise/rbf_kernel.py

@@ -0,0 +1,51 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .... import tensor as mt
+from .core import PairwiseDistances
+from .euclidean import euclidean_distances
+
+
+def rbf_kernel(X, Y=None, gamma=None):
+    """
+    Compute the rbf (gaussian) kernel between X and Y::
+
+        K(x, y) = exp(-gamma ||x-y||^2)
+
+    for each pair of rows x in X and y in Y.
+
+    Read more in the :ref:`User Guide <rbf_kernel>`.
+
+    Parameters
+    ----------
+    X : tensor of shape (n_samples_X, n_features)
+
+    Y : tensor of shape (n_samples_Y, n_features)
+
+    gamma : float, default None
+        If None, defaults to 1.0 / n_features
+
+    Returns
+    -------
+    kernel_matrix : tensor of shape (n_samples_X, n_samples_Y)
+    """
+
+    X, Y = PairwiseDistances.check_pairwise_arrays(X, Y)
+    if gamma is None:
+        gamma = 1.0 / X.shape[1]
+
+    K = euclidean_distances(X, Y, squared=True)
+    K *= -gamma
+    K = mt.exp(K)
+    return K

maxframe/learn/metrics/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/metrics/tests/test_scorer.py

@@ -0,0 +1,26 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from sklearn.metrics import r2_score
+
+from .. import get_scorer
+
+
+def test_get_scorer():
+    with pytest.raises(ValueError):
+        get_scorer("unknown")
+
+    assert get_scorer("r2") is not None
+    assert get_scorer(r2_score) is not None

maxframe/learn/utils/__init__.py

@@ -16,4 +16,4 @@ from .core import convert_to_tensor_or_dataframe
 from .multiclass import check_classification_targets
 from .shuffle import shuffle
 from .sparsefuncs import count_nonzero
-from .validation import check_consistent_length
+from .validation import check_array, check_consistent_length

maxframe/learn/utils/checks.py

@@ -20,7 +20,7 @@ from ... import opcodes
 from ... import tensor as mt
 from ...config import options
 from ...core import ENTITY_TYPE, EntityData, OutputType, get_output_types
-from ...core.operator import Operator, OperatorStage
+from ...core.operator import Operator
 from ...serialization.serializables import (
     BoolField,
     DataTypeField,
@@ -56,7 +56,6 @@ class CheckBase(Operator, LearnOperatorMixin):
         # output input if value not specified
         self.value = value = value if value is not None else x
         self.output_types = get_output_types(value)
-        self.stage = OperatorStage.agg
         return self.new_tileable([x, value], kws=[value.params])
 
 

maxframe/learn/utils/core.py

@@ -14,9 +14,19 @@
 
 import math
 import numbers
+import warnings
 
+import numpy as np
 import pandas as pd
 
+from ...utils import parse_readable_size
+
+try:
+    from sklearn import get_config as sklearn_get_config
+except ImportError:
+    sklearn_get_config = None
+
+from ...config import options
 from ...dataframe import DataFrame, Series
 from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
 from ...tensor import tensor as astensor
@@ -60,3 +70,52 @@ def is_scalar_nan(x):
     False
     """
     return isinstance(x, numbers.Real) and math.isnan(x)
+
+
+def get_chunk_n_rows(row_bytes, max_n_rows=None, working_memory=None):
+    """Calculates how many rows can be processed within working_memory
+
+    Parameters
+    ----------
+    row_bytes : int
+        The expected number of bytes of memory that will be consumed
+        during the processing of each row.
+    max_n_rows : int, optional
+        The maximum return value.
+    working_memory : int or float, optional
+        The number of rows to fit inside this number of MiB will be returned.
+        When None (default), the value of
+        ``sklearn.get_config()['working_memory']`` is used.
+
+    Returns
+    -------
+    int or the value of n_samples
+
+    Warns
+    -----
+    Issues a UserWarning if ``row_bytes`` exceeds ``working_memory`` MiB.
+    """
+
+    if working_memory is None:  # pragma: no cover
+        working_memory = options.learn.working_memory
+        if working_memory is None and sklearn_get_config is not None:
+            working_memory = sklearn_get_config()["working_memory"]
+        elif working_memory is None:
+            working_memory = 1024
+
+    if isinstance(working_memory, int):
+        working_memory *= 2**20
+    else:
+        working_memory = parse_readable_size(working_memory)[0]
+
+    chunk_n_rows = int(working_memory // row_bytes)
+    if max_n_rows is not None:
+        chunk_n_rows = min(chunk_n_rows, max_n_rows)
+    if chunk_n_rows < 1:  # pragma: no cover
+        warnings.warn(
+            "Could not adhere to working_memory config. "
+            "Currently %.0fMiB, %.0fMiB required."
+            % (working_memory, np.ceil(row_bytes * 2**-20))
+        )
+        chunk_n_rows = 1
+    return chunk_n_rows

maxframe/learn/utils/extmath.py

@@ -49,6 +49,19 @@ def _safe_accumulator_op(op, x, *args, **kwargs):
     return result
 
 
+def logsumexp_real(a, axis=None, keepdims=False):
+    """Simplified logsumexp for real arrays without biases"""
+    from ... import tensor as mt
+
+    x = mt.tensor(a)
+    x_max = mt.amax(a, axis=axis, keepdims=True)
+    exp_x_shifted = mt.exp(x - x_max)
+    ret = mt.log(mt.sum(exp_x_shifted, axis=axis, keepdims=True)) + x_max
+    if keepdims:
+        return ret
+    return mt.squeeze(ret, axis=1)
+
+
 def _incremental_mean_and_var(
     X, last_mean, last_variance, last_sample_count, sample_weight=None
 ):
@@ -174,3 +187,27 @@ def _incremental_mean_and_var(
         updated_variance = updated_unnormalized_variance / updated_sample_count
 
     return updated_mean, updated_variance, updated_sample_count
+
+
+def row_norms(X, squared=False):
+    """Row-wise (squared) Euclidean norm of X.
+
+    Performs no input validation.
+
+    Parameters
+    ----------
+    X : array_like
+        The input tensor
+    squared : bool, optional (default = False)
+        If True, return squared norms.
+
+    Returns
+    -------
+    array_like
+        The row-wise (squared) Euclidean norm of X.
+    """
+
+    norms = (X**2).sum(axis=1)
+    if not squared:
+        norms = mt.sqrt(norms)
+    return norms

maxframe/learn/utils/odpsio.py

@@ -0,0 +1,193 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List, NamedTuple, Optional
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, EntityData, OutputType
+from ...core.operator import ObjectOperator
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int32Field,
+    StringField,
+)
+from ...utils import find_objects, replace_objects
+from ..core import LearnOperatorMixin
+
+
+class ToODPSModel(ObjectOperator, LearnOperatorMixin):
+    _op_type_ = opcodes.TO_ODPS_MODEL
+
+    model_name = StringField("model_name", default=None)
+    model_version = StringField("model_version", default=None)
+    training_info = AnyField("training_info", default=None)
+    params = AnyField("params", default=None)
+    format = StringField("format", default=None)
+    lifecycle = Int32Field("lifecycle", default=None)
+    version_lifecycle = Int32Field("version_lifecycle", default=None)
+    description = StringField("description", default=None)
+    version_description = StringField("version_description", default=None)
+    create_model = BoolField("create_model", default=True)
+    set_default_version = BoolField("set_default_version", default=True)
+    location = StringField("location", default=None)
+    storage_options = DictField("storage_options", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(_output_types=[OutputType.object], **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "ToODPSModel", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+
+        if isinstance(op.training_info, ENTITY_TYPE):
+            has_training_info = True
+            op.training_info = inputs[0]
+        else:
+            has_training_info = False
+
+        tileables = find_objects([op.params], ENTITY_TYPE)
+        param_pos = int(has_training_info)
+        replaces = dict(zip(tileables, inputs[param_pos:]))
+        [op.params] = replace_objects([op.params], replaces)
+
+    def __call__(self, training_info, params):
+        inputs = []
+        if isinstance(training_info, ENTITY_TYPE):
+            inputs.append(training_info)
+
+        self.training_info = training_info
+        self.params = params
+        inputs.extend(find_objects([params], ENTITY_TYPE))
+        return self.new_tileable(inputs, shape=())
+
+
+class ToODPSModelMixin:
+    class ODPSModelInfo(NamedTuple):
+        model_format: str
+        model_params: Any
+
+    def _get_odps_model_info(self) -> ODPSModelInfo:
+        raise NotImplementedError
+
+    def to_odps_model(
+        self,
+        model_name: str = None,
+        model_version: str = None,
+        schema: str = None,
+        project: str = None,
+        lifecycle: Optional[int] = None,
+        version_lifecycle: Optional[int] = None,
+        description: Optional[str] = None,
+        version_description: Optional[str] = None,
+        create_model: bool = True,
+        set_default_version: bool = False,
+        location: Optional[str] = None,
+        storage_options: Dict[str, Any] = None,
+    ):
+        """
+        Save trained model to MaxCompute.
+
+        Parameters
+        ----------
+        model_name : str, optional
+            Name of the model. Can be a fully qualified name with format
+            "project.schema.model" or just "model" if project and schema are
+            specified separately.
+        model_version : str, optional
+            Version of the model. If not provided, a default version will be used.
+        schema : str, optional
+            Schema name where the model will be stored. If not provided and
+            project is specified, "default" schema will be used.
+        project : str, optional
+            Project name where the model will be stored.
+        lifecycle : int, optional
+            Lifecycle of the model in days. After this period, the model will
+            be automatically deleted.
+        version_lifecycle : int, optional
+            Lifecycle of the model version in days. After this period, the
+            model version will be automatically deleted.
+        description : str, optional
+            Description of the model.
+        version_description : str, optional
+            Description of the model version.
+        create_model : bool, default True
+            Whether to create the model if it doesn't exist.
+        set_default_version : bool, default False
+            Whether to set this version as the default version of the model.
+        location : str, optional
+            Storage location for the model. If specified, the model can be stored
+            into a customized location. Can be an OSS path with format
+            oss://endpoint/bucket/path.
+        storage_options : dict, optional
+            Extra options for storage, such as role_arn or policy for OSS storage.
+
+        Returns
+        -------
+        Scalar
+            A scalar that can be executed to save the model.
+
+        Examples
+        --------
+        First we fit an XGBoost model.
+
+        >>> import maxframe.dataframe as md
+        >>> from maxframe.learn.datasets import make_classification
+        >>> from maxframe.learn.contrib.xgboost import XGBClassifier
+        >>> X, y = make_classification(1000, n_features=10, n_classes=2)
+        >>> cols = [f"f{idx}" for idx in range(10)]
+        >>> clf = XGBClassifier(n_estimators=10)
+        >>> X_df = md.DataFrame(X, columns=cols)
+        >>> clf.fit(X_df, y)
+
+        Trigger execution and save model with fully qualified name.
+
+        >>> clf.to_odps_model(model_name="project.schema.my_model",
+        ...                   model_version="v1.0").execute()
+
+        You can also save model with a customized path. Need to change `<my_bucket>`
+        and `<user_id>` into your own bucket and user ID.
+
+        >>> clf.to_odps_model(model_name="project.schema.my_model",
+        ...                   model_version="v1.0",
+        ...                   location="oss://oss-cn-shanghai.aliyuncs.com/<my_bucket>/model_name",
+        ...                   storage_options={
+        ...                       "role_arn": "acs:ram::<user_id>:role/aliyunodpsdefaultrole"
+        ...                   }).execute()
+        """
+        if "." not in model_name:
+            if project and not schema:
+                schema = "default"
+            if schema:
+                model_name = f"{schema}.{model_name}"
+            if project:
+                model_name = f"{project}.{model_name}"
+
+        model_info = self._get_odps_model_info()
+
+        op = ToODPSModel(
+            model_name=model_name,
+            model_version=model_version,
+            format=model_info.model_format,
+            lifecycle=lifecycle,
+            version_lifecycle=version_lifecycle,
+            description=description,
+            version_description=version_description,
+            create_model=create_model,
+            set_default_version=set_default_version,
+            location=location,
+            storage_options=storage_options,
+        )
+        return op(getattr(self, "training_info_"), model_info.model_params)

maxframe/learn/utils/validation.py

@@ -695,9 +695,9 @@ def _check_sample_weight(
         dtype = np.float64
 
     if sample_weight is None:
-        sample_weight = np.ones(n_samples, dtype=dtype)
+        sample_weight = mt.ones(n_samples, dtype=dtype)
     elif isinstance(sample_weight, numbers.Number):
-        sample_weight = np.full(n_samples, sample_weight, dtype=dtype)
+        sample_weight = mt.full(n_samples, sample_weight, dtype=dtype)
     else:
         if dtype is None:
             dtype = [np.float64, np.float32]

maxframe/lib/compat.py

@@ -1,3 +1,5 @@
+import asyncio
+import functools
 from typing import TYPE_CHECKING, Callable, List, Sequence, Tuple, Union
 
 import numpy as np
@@ -143,3 +145,41 @@ def case_when(
 def patch_pandas():
     if not hasattr(pd.Series, "case_when"):
         pd.Series.case_when = case_when
+
+
+class cached_property:
+    """
+    A property that is only computed once per instance and then replaces itself
+    with an ordinary attribute. Deleting the attribute resets the property.
+    Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
+    """  # noqa
+
+    def __init__(self, func):
+        self.__doc__ = getattr(func, "__doc__")
+        self.func = func
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+
+        if asyncio.iscoroutinefunction(self.func):
+            return self._wrap_in_coroutine(obj)
+
+        value = obj.__dict__[self.func.__name__] = self.func(obj)
+        return value
+
+    def _wrap_in_coroutine(self, obj):
+        @functools.wraps(obj)
+        def wrapper():
+            future = asyncio.ensure_future(self.func(obj))
+            obj.__dict__[self.func.__name__] = future
+            return future
+
+        return wrapper()
+
+
+# isort: off
+try:
+    from functools import cached_property  # noqa: F811, F401
+except ImportError:
+    pass

maxframe/lib/dtypes_extension/__init__.py

@@ -12,4 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .dtypes import ArrowDtype, dict_, is_list_dtype, is_map_dtype, list_
+from .blob import (
+    ArrowBlobType,
+    ExternalBlobDtype,
+    ExternalBlobExtensionArray,
+    SolidBlob,
+    new_blob,
+)
+from .dtypes import (
+    ArrowDtype,
+    dict_,
+    is_list_dtype,
+    is_map_dtype,
+    is_struct_dtype,
+    list_,
+    struct_,
+)