maxframe 1.3.0__cp310-cp310-win32.whl → 2.0.0b1__cp310-cp310-win32.whl

maxframe/learn/contrib/lightgbm/dataset.py

@@ -0,0 +1,153 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from .... import opcodes
+from ....core import EntityData
+from ....core.entity.output_types import get_output_types
+from ....core.operator.base import Operator
+from ....core.operator.core import TileableOperatorMixin
+from ....dataframe.core import DATAFRAME_TYPE
+from ....serialization.serializables import BoolField, KeyField, ListField
+from ....serialization.serializables.field import AnyField
+from ....tensor import tensor as astensor
+from ....tensor.core import TENSOR_TYPE
+from ....typing_ import TileableType
+from ...utils import convert_to_tensor_or_dataframe
+
+
+class ToLGBMDataset(Operator, TileableOperatorMixin):
+    _op_type_ = opcodes.TO_LGBM_DATASET
+
+    data = KeyField("data", default=None)
+    label = KeyField("label", default=None)
+    reference = KeyField("reference", default=None)
+    weight = KeyField("weight", default=None)
+    init_score = KeyField("init_score", default=None)
+    group = AnyField("group", default=None)
+    feature_name = ListField("feature_name", default=None)
+    categorical_feature = ListField("categorical_feature", default=None)
+    params = AnyField("params", default=None)
+    free_raw_data = BoolField("free_raw_data", default=None)
+    position = AnyField("position", default=None)
+    # if to collocate the data, label and weight
+    collocate = BoolField("collocate", default=False)
+
+    @property
+    def output_limit(self):
+        if self.collocate:
+            return 1 + sum(bool(x) for x in [self.label, self.weight, self.init_score])
+        return 1
+
+    @classmethod
+    def _set_inputs(cls, op: "ToLGBMDataset", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.data is not None:
+            op.data = op._inputs[0]
+        has_label = op.label is not None
+        if has_label:
+            op.label = op._inputs[1]
+        if op.weight is not None:
+            i = 1 if not has_label else 2
+            op.weight = op._inputs[i]
+        if op.init_score is not None:
+            op.init_score = op._inputs[-1]
+
+    @staticmethod
+    def _get_kw(obj):
+        if isinstance(obj, TENSOR_TYPE):
+            return {"shape": obj.shape, "dtype": obj.dtype, "order": obj.order}
+        else:
+            return {
+                "shape": obj.shape,
+                "dtypes": obj.dtypes,
+                "index_value": obj.index_value,
+                "columns_value": obj.columns_value,
+            }
+
+    def __call__(self):
+        inputs = [self.data]
+        kw = self._get_kw(self.data)
+        if self.label is not None:
+            inputs.append(self.label)
+        if self.weight is not None:
+            inputs.append(self.weight)
+        if self.init_score is not None:
+            inputs.append(self.init_score)
+
+        return self.new_tileable(inputs, **kw)
+
+
+def check_data(data):
+    data = convert_to_tensor_or_dataframe(data)
+    if data.ndim != 2:
+        raise ValueError(f"Expecting 2-d data, got: {data.ndim}-d")
+
+    return data
+
+
+def check_array_like(y: TileableType, name: str) -> TileableType:
+    if y is None:
+        return
+    y = convert_to_tensor_or_dataframe(y)
+    if isinstance(y, DATAFRAME_TYPE):
+        y = y.iloc[:, 0]
+    return astensor(y)
+
+
+def to_lgbm_dataset(
+    data,
+    label=None,
+    reference=None,
+    weight=None,
+    group=None,
+    init_score=None,
+    feature_name="auto",
+    categorical_feature="auto",
+    params=None,
+    free_raw_data=True,
+    position=None,
+):
+    data = check_data(data)
+    label = check_array_like(label, "label")
+    weight = check_array_like(weight, "weight")
+    init_score = check_array_like(init_score, "init_score")
+
+    if weight is not None and weight.ndim > 1:
+        raise ValueError("weight must be 1-dimensional")
+
+    # If not multiple outputs, try to collect the chunks on same worker into one
+    # to feed the data into LightGBM for training.
+    op = ToLGBMDataset(
+        data=data,
+        label=label,
+        reference=reference,
+        weight=weight,
+        group=group,
+        init_score=init_score,
+        feature_name=None if feature_name == "auto" else feature_name,
+        categorical_feature=None
+        if categorical_feature == "auto"
+        else categorical_feature,
+        params=params,
+        free_raw_data=free_raw_data,
+        position=position,
+        gpu=data.op.gpu,
+        _output_types=get_output_types(data),
+    )
+    return op()
+
+
+Dataset = to_lgbm_dataset

maxframe/learn/contrib/lightgbm/regressor.py

@@ -0,0 +1,29 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..utils import make_import_error_func
+from .core import LGBMScikitLearnBase
+
+try:
+    import lightgbm
+except ImportError:
+    lightgbm = None
+
+
+if not lightgbm:
+    LGBMRegressor = make_import_error_func("lightgbm")
+else:
+
+    class LGBMRegressor(LGBMScikitLearnBase, lightgbm.LGBMRegressor):
+        _default_objective = "regression"

maxframe/learn/contrib/lightgbm/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/contrib/lightgbm/tests/test_callback.py

@@ -0,0 +1,58 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+
+import pytest
+
+from ..callback import (
+    EarlyStoppingCallback,
+    LGBMTrainingCallback,
+    early_stopping,
+    reset_parameter,
+)
+
+try:
+    from lightgbm.callback import _EarlyStoppingCallback
+except ImportError:
+    try:
+        from lightgbm.callback import early_stopping as _EarlyStoppingCallback
+    except ImportError:
+        pytestmark = pytest.mark.skip("Need lightgbm to run the test")
+
+
+def test_lgbm_training_callbacks():
+    early_stopping_cb = early_stopping(stopping_rounds=5)
+    assert not early_stopping_cb.has_custom_code()
+
+    local_early_stopping_cb = early_stopping_cb.to_local()
+    if isinstance(_EarlyStoppingCallback, type):
+        assert isinstance(local_early_stopping_cb, _EarlyStoppingCallback)
+        assert local_early_stopping_cb.stopping_rounds == 5
+    else:
+        assert local_early_stopping_cb.__qualname__.startswith(
+            _EarlyStoppingCallback.__name__
+        )
+        cb_vars = inspect.getclosurevars(local_early_stopping_cb).nonlocals
+        assert cb_vars["stopping_rounds"] == 5
+
+    rmt_early_stopping_cb = LGBMTrainingCallback.from_local(local_early_stopping_cb)
+    assert isinstance(rmt_early_stopping_cb, EarlyStoppingCallback)
+    assert rmt_early_stopping_cb.stopping_rounds == 5
+
+    lr_sch_cb = reset_parameter(a=[0.5, 0.4, 0.1])
+    assert not lr_sch_cb.has_custom_code()
+
+    lr_sch_cb = reset_parameter(a=lambda x: x * 0.1)
+    assert lr_sch_cb.has_custom_code()

maxframe/learn/contrib/llm/models/dashscope.py

@@ -34,8 +34,25 @@ class DashScopeLLMMixin(Serializable):
 
 
 class DashScopeTextLLM(TextLLM, DashScopeLLMMixin):
+    """
+    DashScope text LLM.
+    """
+
     api_key_resource = StringField("api_key_resource", default=None)
 
+    def __init__(self, name: str, api_key_resource: str):
+        """
+        Initialize a DashScope text LLM.
+
+        Parameters
+        ----------
+        name : str
+            The LLM name to use, check DashScope for `available models <https://help.aliyun.com/zh/model-studio/getting-started/models>`_.
+        api_key_resource : str
+            The MaxCompute resource file name containing the DashScope API key.
+        """
+        super().__init__(name=name, api_key_resource=api_key_resource)
+
     def generate(
         self,
         data,
@@ -50,8 +67,25 @@ class DashScopeTextLLM(TextLLM, DashScopeLLMMixin):
 
 
 class DashScopeMultiModalLLM(MultiModalLLM, DashScopeLLMMixin):
+    """
+    DashScope multi-modal LLM.
+    """
+
     api_key_resource = StringField("api_key_resource", default=None)
 
+    def __init__(self, name: str, api_key_resource: str):
+        """
+        Initialize a DashScope multi-modal LLM.
+
+        Parameters
+        ----------
+        name : str
+            The LLM name to use, check DashScope for `available models <https://help.aliyun.com/zh/model-studio/getting-started/models>`_.
+        api_key_resource : str
+            The MaxCompute resource file name containing the DashScope API key.
+        """
+        super().__init__(name=name, api_key_resource=api_key_resource)
+
     def generate(
         self,
         data,

maxframe/learn/contrib/llm/models/managed.py

@@ -27,6 +27,21 @@ class ManagedLLMTextGenOperator(LLMTextGenOperator):
 
 
 class ManagedTextLLM(TextLLM):
+    """
+    Managed text LLM by MaxFrame.
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialize a managed text LLM.
+
+        Parameters
+        ----------
+        name : str
+            The managed text LLM name to use.
+        """
+        super().__init__(name=name)
+
     def generate(
         self,
         data,

maxframe/learn/contrib/llm/multi_modal.py

@@ -34,6 +34,98 @@ def generate(
     prompt_template: Dict[str, Any],
     params: Dict[str, Any] = None,
 ):
+    """
+    Generate text with multi model llm based on given data and prompt template.
+
+    Parameters
+    ----------
+    data : DataFrame or Series
+        Input data used for generation. Can be maxframe DataFrame, Series that contain text to be processed.
+    model : MultiModalLLM
+        Language model instance support **MultiModal** inputs used for text generation.
+    prompt_template : List[Dict[str, List[Dict[str, str]]]]
+        List of message with column names as placeholders. Each message contains a role and content. Content is a list of dict, each dict contains a text or image, the value can reference column data from input.
+
+        Here is an example of prompt template.
+
+        .. code-block:: python
+
+            [
+                {
+                    "role": "<role>",  # e.g. "user" or "assistant"
+                    "content": [
+                        {
+                            # At least one of these fields is required
+                            "image": "<image_data_url>",  # optional
+                            "text": "<prompt_text_template>"  # optional
+                        },
+                        ...
+                    ]
+                }
+            ]
+
+        Where:
+
+        - ``text`` can be a Python format string using column names from input data as parameters (e.g. ``"{column_name}"``)
+        - ``image`` should be a DataURL string following `RFC2397 <https://en.wikipedia.org/wiki/Data_URI_scheme>`_ standard with format.
+
+        .. code-block:: none
+
+            data:<mime_type>[;base64],<column_name>
+
+
+    params : Dict[str, Any], optional
+        Additional parameters for generation configuration, by default None.
+        Can include settings like temperature, max_tokens, etc.
+
+    Returns
+    -------
+    DataFrame
+        Generated text raw response and success status. If the success is False, the generated text will return the
+        error message.
+
+    Notes
+    -----
+    - The ``api_key_resource`` parameter should reference a text file resource in MaxCompute that contains only your DashScope API key.
+
+    - Using DashScope services requires enabling public network access for your MaxCompute project. This can be configured through the MaxCompute console by `enabling the Internet access feature <https://help.aliyun.com/zh/maxcompute/user-guide/network-connection-process>`_ for your project. Without this configuration, the API calls to DashScope will fail due to network connectivity issues.
+
+    Examples
+    --------
+    You can initialize a DashScope multi-modal model (such as qwen-vl-max) by providing a model name and an ``api_key_resource``.
+    The ``api_key_resource`` is a MaxCompute resource name that points to a text file containing a `DashScope <https://dashscope.aliyun.com/>`_ API key.
+
+    >>> from maxframe.learn.contrib.llm.models.dashscope import DashScopeMultiModalLLM
+    >>> import maxframe.dataframe as md
+    >>>
+    >>> model = DashScopeMultiModalLLM(
+    ...     name="qwen-vl-max",
+    ...     api_key_resource="<api-key-resource-name>"
+    ... )
+
+    We use Data Url Schema to provide multi modal input in prompt template, here is an example to fill in the image from table.
+
+    Assuming you have a MaxCompute table with two columns: ``image_id`` (as the index) and ``encoded_image_data_base64`` (containing Base64 encoded image data),
+    you can construct a prompt message template as follows:
+
+    >>> df = md.read_odps_table("image_content", index_col="image_id")
+
+    >>> prompt_template = [
+    ...     {
+    ...         "role": "user",
+    ...         "content": [
+    ...             {
+    ...                 "image": "data:image/png;base64,encoded_image_data_base64",
+    ...             },
+    ...             {
+    ...                 "text": "Analyze this image in detail",
+    ...             },
+    ...         ],
+    ...     },
+    ... ]
+    >>> result = model.generate(df, prompt_template)
+    >>> result.execute()
+    """
     if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
         raise ValueError("data must be a maxframe dataframe or series object")
     if not isinstance(model, MultiModalLLM):

maxframe/learn/contrib/llm/text.py

@@ -145,16 +145,16 @@ def generate(
     >>> import maxframe.dataframe as md
     >>>
     >>> # Initialize the model
-    >>> llm = ManagedTextLLM(name="Qwen2.5-1.5B")
+    >>> llm = ManagedTextLLM(name="Qwen2.5-0.5B-instruct")
     >>>
     >>> # Prepare prompt template
     >>> messages = [
     ...     {
     ...         "role": "user",
-    ...         "content": "{query}",
+    ...         "content": "Help answer following question: {query}",
     ...     },
     ... ]
-    >>>
+
     >>> # Create sample data
     >>> df = md.DataFrame({"query": ["What is machine learning?"]})
     >>>
@@ -177,7 +177,7 @@ def summary(series, model: TextLLM, index=None):
 
     Parameters
     ----------
-    series : pandas.Series
+    series : Series
         A maxframe Series containing text data to be summarized.
         Each element should be a text string.
     model : TextLLM
@@ -189,6 +189,11 @@ def summary(series, model: TextLLM, index=None):
     -------
     maxframe.Series
         A pandas Series containing the generated summaries and success status.
+
+    Notes
+    -----
+      **Preview:** This API is in preview state and may be unstable.
+      The interface may change in future releases.
     """
     if not isinstance(series, Series):
         raise ValueError("series must be a maxframe series object")
@@ -208,7 +213,7 @@ def translate(
     Parameters
     ----------
     series : pandas.Series
-        A maxframe Series containing text data to be translate.
+        A maxframe Series containing text data to translate.
         Each element should be a text string.
     model : TextLLM
         Language model instance used for text summarization.
@@ -223,6 +228,12 @@ def translate(
     -------
     maxframe.Series
         A pandas Series containing the generated translation and success status.
+
+    Notes
+    -----
+      **Preview:** This API is in preview state and may be unstable.
+      The interface may change in future releases.
+
     """
     if not isinstance(series, Series):
         raise ValueError("series must be a maxframe series object")
@@ -268,6 +279,11 @@ def classify(
     -------
     maxframe.Series
         A pandas Series containing the generated classification results and success status.
+
+    Notes
+    -----
+      **Preview:** This API is in preview state and may be unstable.
+      The interface may change in future releases.
     """
     if not isinstance(series, Series):
         raise ValueError("series must be a maxframe series object")

maxframe/learn/contrib/models.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Type
+
 from ... import opcodes
 from ...core import ENTITY_TYPE, OutputType
 from ...core.operator import ObjectOperator, ObjectOperatorMixin
@@ -21,7 +23,30 @@ from ...serialization.serializables import (
     FunctionField,
     TupleField,
 )
+from ...udf import BuiltinFunction
 from ...utils import find_objects, replace_objects
+from ..core import Model, ModelData
+
+
+class ModelWithEvalData(ModelData):
+    __slots__ = ("_evals_result",)
+
+    _evals_result: dict
+
+    def __init__(self, *args, evals_result=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._evals_result = evals_result if evals_result is not None else dict()
+
+    def execute(self, session=None, **kw):
+        # The evals_result should be fetched when BoosterData.execute() is called.
+        result = super().execute(session=session, **kw)
+        if self.op.has_evals_result and self.key == self.op.outputs[0].key:
+            self._evals_result.update(self.op.outputs[1].fetch(session=session))
+        return result
+
+
+class ModelWithEval(Model):
+    pass
 
 
 class ModelDataSource(ObjectOperator, ObjectOperatorMixin):
@@ -29,7 +54,7 @@ class ModelDataSource(ObjectOperator, ObjectOperatorMixin):
 
     data = AnyField("data")
 
-    def __call__(self, model_cls):
+    def __call__(self, model_cls: Type[ModelWithEval]):
         self._output_types = [OutputType.object]
         return self.new_tileable(None, object_class=model_cls)
 
@@ -48,14 +73,18 @@ class ModelApplyChunk(ObjectOperator, ObjectOperatorMixin):
         self._output_types = list(output_types)
         super().__init__(**kwargs)
 
-    def _set_inputs(self, inputs):
-        super()._set_inputs(inputs)
-        old_inputs = find_objects(self.args, ENTITY_TYPE) + find_objects(
-            self.kwargs, ENTITY_TYPE
+    def has_custom_code(self) -> bool:
+        return not isinstance(self.func, BuiltinFunction)
+
+    @classmethod
+    def _set_inputs(cls, op: "ModelApplyChunk", inputs):
+        super()._set_inputs(op, inputs)
+        old_inputs = find_objects(op.args, ENTITY_TYPE) + find_objects(
+            op.kwargs, ENTITY_TYPE
         )
-        mapping = {o: n for o, n in zip(old_inputs, self._inputs[1:])}
-        self.args = replace_objects(self.args, mapping)
-        self.kwargs = replace_objects(self.kwargs, mapping)
+        mapping = {o: n for o, n in zip(old_inputs, op._inputs[1:])}
+        op.args = replace_objects(op.args, mapping)
+        op.kwargs = replace_objects(op.kwargs, mapping)
 
     @property
     def output_limit(self) -> int:
@@ -72,6 +101,6 @@ class ModelApplyChunk(ObjectOperator, ObjectOperatorMixin):
         return self.new_tileables(inputs, kws=output_kws)
 
 
-def to_remote_model(model, model_cls):
+def to_remote_model(model, model_cls: Type[ModelWithEval]) -> ModelWithEval:
     op = ModelDataSource(data=model)
     return op(model_cls)

maxframe/learn/contrib/utils.py

@@ -14,6 +14,8 @@
 
 import sys
 
+from ...serialization.serializables import Serializable
+
 
 def make_import_error_func(package_name):
     def _func(*_, **__):  # pragma: no cover
@@ -51,3 +53,56 @@ def config_mod_getattr(mod_dict, globals_):
             "__warningregistry__": dict(),
         }
     )
+
+
+class TrainingCallback(Serializable):
+    _local_cls = None
+
+    @classmethod
+    def _load_local_to_remote_mapping(cls, globals_dict):
+        if cls._local_to_remote:
+            return
+        for v in globals_dict.values():
+            if isinstance(v, type) and issubclass(v, cls) and v._local_cls is not None:
+                cls._local_to_remote[v._local_cls] = v
+
+    @classmethod
+    def from_local(cls, callback_obj):
+        if isinstance(callback_obj, (list, tuple)):
+            return [cls.from_local(x) for x in callback_obj]
+        if not type(callback_obj) in cls._local_to_remote:
+            return callback_obj
+
+        kw = {}
+        remote_cls = cls._local_to_remote[type(callback_obj)]
+        for attr in remote_cls._FIELDS:
+            try:
+                kw[attr] = getattr(callback_obj, attr)
+            except AttributeError:
+                pass
+        return remote_cls(**kw)
+
+    def has_custom_code(self) -> bool:
+        return False
+
+    @classmethod
+    def remote_to_local(cls, remote_obj):
+        if isinstance(remote_obj, (list, tuple)):
+            return [cls.remote_to_local(x) for x in remote_obj]
+        if not isinstance(remote_obj, TrainingCallback):
+            return remote_obj
+        return remote_obj.to_local()
+
+    def _extract_kw(self) -> dict:
+        kw = {}
+        for attr in type(self)._FIELDS:
+            val = getattr(self, attr, None)
+            if val is not None:
+                kw[attr] = val
+        return kw
+
+    def to_local(self):
+        return type(self)._local_cls(**self._extract_kw())
+
+    def __call__(self, *args, **kwargs):
+        return self.to_local()(*args, **kwargs)