maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/learn/utils/odpsio.py

@@ -0,0 +1,262 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List, NamedTuple, Optional, Set
+
+from odps import ODPS
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, EntityData, OutputType
+from ...core.operator import ObjectOperator
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    Int32Field,
+    StringField,
+)
+from ...utils import find_objects, replace_objects
+from ..core import LearnOperatorMixin
+
+_odps_model_classes: Set["ODPSModelMixin"] = set()
+
+
+def register_odps_model(model_cls: "ODPSModelMixin"):
+    _odps_model_classes.add(model_cls)
+    return model_cls
+
+
+class ReadODPSModel(ObjectOperator, LearnOperatorMixin):
+    _op_type_ = opcodes.READ_ODPS_MODEL
+
+    model_name = StringField("model_name", default=None)
+    model_version = StringField("model_version", default=None)
+    format = StringField("format", default=None)
+    location = StringField("location", default=None)
+    storage_options = DictField("storage_options", default=None)
+
+    def has_custom_code(self) -> bool:
+        return True
+
+    def __call__(self):
+        if not self.format.startswith("BOOSTED_TREE_"):
+            # todo support more model formats
+            raise ValueError("Only support boosted tree format")
+        for model_cls in _odps_model_classes:
+            ret = model_cls._build_odps_source_model(self)
+            if ret is not None:
+                return ret
+        raise ValueError(f"Model {self.model_name} not supported")
+
+
+class ToODPSModel(ObjectOperator, LearnOperatorMixin):
+    _op_type_ = opcodes.TO_ODPS_MODEL
+
+    model_name = StringField("model_name", default=None)
+    model_version = StringField("model_version", default=None)
+    training_info = AnyField("training_info", default=None)
+    params = AnyField("params", default=None)
+    format = StringField("format", default=None)
+    lifecycle = Int32Field("lifecycle", default=None)
+    version_lifecycle = Int32Field("version_lifecycle", default=None)
+    description = StringField("description", default=None)
+    version_description = StringField("version_description", default=None)
+    create_model = BoolField("create_model", default=True)
+    set_default_version = BoolField("set_default_version", default=True)
+    location = StringField("location", default=None)
+    storage_options = DictField("storage_options", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(_output_types=[OutputType.object], **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "ToODPSModel", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+
+        if isinstance(op.training_info, ENTITY_TYPE):
+            has_training_info = True
+            op.training_info = inputs[0]
+        else:
+            has_training_info = False
+
+        tileables = find_objects([op.params], ENTITY_TYPE)
+        param_pos = int(has_training_info)
+        replaces = dict(zip(tileables, inputs[param_pos:]))
+        [op.params] = replace_objects([op.params], replaces)
+
+    def __call__(self, training_info, params):
+        inputs = []
+        if isinstance(training_info, ENTITY_TYPE):
+            inputs.append(training_info)
+
+        self.training_info = training_info
+        self.params = params
+        inputs.extend(find_objects([params], ENTITY_TYPE))
+        return self.new_tileable(inputs, shape=())
+
+
+class ODPSModelMixin:
+    class ODPSModelInfo(NamedTuple):
+        model_format: str
+        model_params: Any
+
+    @classmethod
+    def _build_odps_source_model(cls, op: ReadODPSModel) -> Any:
+        return None
+
+    def _get_odps_model_info(self) -> ODPSModelInfo:
+        raise NotImplementedError
+
+    def to_odps_model(
+        self,
+        model_name: str,
+        model_version: str = None,
+        schema: str = None,
+        project: str = None,
+        lifecycle: Optional[int] = None,
+        version_lifecycle: Optional[int] = None,
+        description: Optional[str] = None,
+        version_description: Optional[str] = None,
+        create_model: bool = True,
+        set_default_version: bool = False,
+        location: Optional[str] = None,
+        storage_options: Dict[str, Any] = None,
+    ):
+        """
+        Save trained model to MaxCompute.
+
+        Parameters
+        ----------
+        model_name : str, optional
+            Name of the model. Can be a fully qualified name with format
+            "project.schema.model" or just "model" if project and schema are
+            specified separately.
+        model_version : str, optional
+            Version of the model. If not provided, a default version will be used.
+        schema : str, optional
+            Schema name where the model will be stored. If not provided and
+            project is specified, "default" schema will be used.
+        project : str, optional
+            Project name where the model will be stored.
+        lifecycle : int, optional
+            Lifecycle of the model in days. After this period, the model will
+            be automatically deleted.
+        version_lifecycle : int, optional
+            Lifecycle of the model version in days. After this period, the
+            model version will be automatically deleted.
+        description : str, optional
+            Description of the model.
+        version_description : str, optional
+            Description of the model version.
+        create_model : bool, default True
+            Whether to create the model if it doesn't exist.
+        set_default_version : bool, default False
+            Whether to set this version as the default version of the model.
+        location : str, optional
+            Storage location for the model. If specified, the model can be stored
+            into a customized location. Can be an OSS path with format
+            oss://endpoint/bucket/path.
+        storage_options : dict, optional
+            Extra options for storage, such as role_arn or policy for OSS storage.
+
+        Returns
+        -------
+        Scalar
+            A scalar that can be executed to save the model.
+
+        Examples
+        --------
+        First we fit an XGBoost model.
+
+        >>> import maxframe.dataframe as md
+        >>> from maxframe.learn.datasets import make_classification
+        >>> from maxframe.learn.contrib.xgboost import XGBClassifier
+        >>> X, y = make_classification(1000, n_features=10, n_classes=2)
+        >>> cols = [f"f{idx}" for idx in range(10)]
+        >>> clf = XGBClassifier(n_estimators=10)
+        >>> X_df = md.DataFrame(X, columns=cols)
+        >>> clf.fit(X_df, y)
+
+        Trigger execution and save model with fully qualified name.
+
+        >>> clf.to_odps_model(model_name="project.schema.my_model",
+        ...                   model_version="v1.0").execute()
+
+        You can also save model with a customized path. Need to change `<my_bucket>`
+        and `<user_id>` into your own bucket and user ID.
+
+        >>> clf.to_odps_model(model_name="project.schema.my_model",
+        ...                   model_version="v1.0",
+        ...                   location="oss://oss-cn-shanghai.aliyuncs.com/<my_bucket>/model_name",
+        ...                   storage_options={
+        ...                       "role_arn": "acs:ram::<user_id>:role/aliyunodpsdefaultrole"
+        ...                   }).execute()
+        """
+        model_name = _build_odps_model_name(model_name, schema, project)
+        model_info = self._get_odps_model_info()
+
+        op = ToODPSModel(
+            model_name=model_name,
+            model_version=model_version,
+            format=model_info.model_format,
+            lifecycle=lifecycle,
+            version_lifecycle=version_lifecycle,
+            description=description,
+            version_description=version_description,
+            create_model=create_model,
+            set_default_version=set_default_version,
+            location=location,
+            storage_options=storage_options,
+        )
+        return op(getattr(self, "training_info_"), model_info.model_params)
+
+
+def _build_odps_model_name(model_name: str, schema: str, project: str = None):
+    if "." not in model_name:
+        if project and not schema:
+            schema = "default"
+        if schema:
+            model_name = f"{schema}.{model_name}"
+        if project:
+            model_name = f"{project}.{model_name}"
+    return model_name
+
+
+def read_odps_model(
+    model_name: str,
+    schema: str = None,
+    project: str = None,
+    model_version: str = None,
+    odps_entry: ODPS = None,
+):
+    odps_entry = odps_entry or ODPS.from_global() or ODPS.from_environments()
+    if not hasattr(odps_entry, "get_model"):
+        raise RuntimeError("Need to install pyodps>=0.11.5 to use read_odps_model")
+
+    model_obj = odps_entry.get_model(model_name, project, schema)
+    if model_version:
+        model_obj = model_obj.versions[model_version]
+    # check if model exists
+    model_obj.reload()
+
+    full_model_name = _build_odps_model_name(model_name, schema, project)
+    location = model_obj.path
+    format = model_obj.type.value
+    op = ReadODPSModel(
+        model_name=full_model_name,
+        model_version=model_version,
+        location=location,
+        format=format,
+    )
+    return op()

maxframe/learn/utils/shuffle.py

@@ -0,0 +1,114 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+from collections.abc import Iterable
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import ExecutableTuple, OutputType, get_output_types
+from ...core.operator import MapReduceOperator
+from ...dataframe.utils import parse_index
+from ...serialization.serializables import FieldTypes, Int64Field, TupleField
+from ...tensor.utils import check_random_state, gen_random_seeds, validate_axis
+from ...utils import tokenize
+from ..core import LearnOperatorMixin
+from . import convert_to_tensor_or_dataframe
+
+
+def _shuffle_index_value(op, index_value, chunk_index=None):
+    key = tokenize((op._values_, chunk_index, index_value.key))
+    return parse_index(pd.Index([], index_value.to_pandas().dtype), key=key)
+
+
+class LearnShuffle(MapReduceOperator, LearnOperatorMixin):
+    _op_type_ = opcodes.PERMUTATION
+
+    axes = TupleField("axes", FieldTypes.int32)
+    seeds = TupleField("seeds", FieldTypes.uint32)
+    n_samples = Int64Field("n_samples", default=None)
+
+    reduce_sizes = TupleField("reduce_sizes", FieldTypes.uint32)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @property
+    def output_limit(self):
+        if self.stage is None:
+            return len(self.output_types)
+        return 1
+
+    def _shuffle_index_value(self, index_value):
+        return _shuffle_index_value(self, index_value)
+
+    def _shuffle_dtypes(self, dtypes):
+        seed = self.seeds[self.axes.index(1)]
+        rs = np.random.RandomState(seed)
+        shuffled_dtypes = dtypes[rs.permutation(np.arange(len(dtypes)))]
+        return shuffled_dtypes
+
+    def _calc_params(self, params):
+        axes = set(self.axes)
+        for i, output_type, param in zip(itertools.count(0), self.output_types, params):
+            if output_type == OutputType.dataframe:
+                if 0 in axes:
+                    param["index_value"] = self._shuffle_index_value(
+                        param["index_value"]
+                    )
+                if 1 in axes:
+                    dtypes = param["dtypes"] = self._shuffle_dtypes(param["dtypes"])
+                    param["columns_value"] = parse_index(dtypes.index, store_data=True)
+            elif output_type == OutputType.series:
+                if 0 in axes:
+                    param["index_value"] = self._shuffle_index_value(
+                        param["index_value"]
+                    )
+            param["_position_"] = i
+        return params
+
+    def __call__(self, arrays):
+        params = self._calc_params([ar.params for ar in arrays])
+        return self.new_tileables(arrays, kws=params)
+
+
+def shuffle(*arrays, random_state=None, n_samples=None, axes=None):
+    arrays = [convert_to_tensor_or_dataframe(ar) for ar in arrays]
+    axes = axes or (0,)
+    if not isinstance(axes, Iterable):
+        axes = (axes,)
+    elif not isinstance(axes, tuple):
+        axes = tuple(axes)
+    random_state = check_random_state(random_state).to_numpy()
+    if n_samples:
+        raise TypeError(f"n_samples argument of shuffle() not supported.")
+
+    max_ndim = max(ar.ndim for ar in arrays)
+    axes = tuple(np.unique([validate_axis(max_ndim, ax) for ax in axes]).tolist())
+    seeds = gen_random_seeds(len(axes), random_state)
+
+    # verify shape
+    for ax in axes:
+        shapes = {ar.shape[ax] for ar in arrays if ax < ar.ndim}
+        if len(shapes) > 1:
+            raise ValueError(f"arrays do not have same shape on axis {ax}")
+
+    op = LearnShuffle(axes=axes, seeds=seeds, output_types=get_output_types(*arrays))
+    shuffled_arrays = op(arrays)
+    if len(arrays) == 1:
+        return shuffled_arrays[0]
+    else:
+        return ExecutableTuple(shuffled_arrays)

maxframe/learn/utils/sparsefuncs.py

@@ -0,0 +1,87 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional
+
+import numpy as np
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import OutputType
+from ...core.operator import Operator, TileableOperatorMixin
+from ...serialization.serializables import Int16Field, ReferenceField
+from ...typing_ import EntityType
+
+
+class LearnCountNonzero(Operator, TileableOperatorMixin):
+    _op_module_ = "learn"
+    _op_type_ = opcodes.COUNT_NONZERO
+
+    axis = Int16Field("axis")
+    sample_weight = ReferenceField("sample_weight")
+
+    @classmethod
+    def _set_inputs(cls, op: "LearnCountNonzero", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        if op.sample_weight is not None:
+            op.sample_weight = inputs[-1]
+
+    def __call__(self, x, sample_weight=None):
+        self.sample_weight = sample_weight
+        self._output_types = [
+            OutputType.scalar if self.axis is None else OutputType.tensor
+        ]
+        dtype = np.dtype(int)
+        inputs = [x]
+        if sample_weight is not None:
+            dtype = sample_weight.dtype
+            inputs = [x, sample_weight]
+
+        if self.axis is None:
+            shape = ()
+        else:
+            shape = (x.shape[1 - self.axis],)
+
+        return self.new_tileable(inputs, shape=shape, dtype=dtype)
+
+
+def count_nonzero(X, axis: Optional[int] = None, sample_weight=None):
+    """A variant of X.getnnz() with extension to weighting on axis 0
+
+    Useful in efficiently calculating multilabel metrics.
+
+    Parameters
+    ----------
+    X : CSR sparse matrix of shape (n_samples, n_labels)
+        Input data.
+
+    axis : None, 0 or 1
+        The axis on which the data is aggregated.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Weight for each row of X.
+    """
+    if axis == -1:
+        axis = 1
+    elif axis == -2:
+        axis = 0
+    if axis is not None and axis not in (0, 1):
+        raise ValueError(f"Unsupported axis: {axis}")
+
+    X = mt.asarray(X)
+    if sample_weight is not None:
+        sample_weight = mt.asarray(sample_weight)
+
+    op = LearnCountNonzero(axis=axis)
+    return op(X, sample_weight=sample_weight)