maxframe 1.2.1__cp311-cp311-win_amd64.whl → 1.3.0__cp311-cp311-win_amd64.whl

maxframe/io/odpsio/schema.py

@@ -28,6 +28,7 @@ from ...protocol import DataFrameTableMeta
 from ...tensor.core import TENSOR_TYPE
 
 _TEMP_TABLE_PREFIX = "tmp_mf_"
+DEFAULT_SINGLE_INDEX_NAME = "_idx_0"
 
 _arrow_to_odps_types = {
     pa.string(): odps_types.string,

maxframe/learn/contrib/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import graph, llm, pytorch
+from . import graph, llm, models, pytorch
 
-del graph
-del llm
-del pytorch
+del graph, llm, models, pytorch

maxframe/learn/contrib/llm/__init__.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from . import models, multi_modal, text
 
 del models

maxframe/learn/contrib/llm/core.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from typing import Any, Dict
 
 import numpy as np
@@ -19,6 +20,8 @@ import pandas as pd
 from ....core.entity.output_types import OutputType
 from ....core.operator.base import Operator
 from ....core.operator.core import TileableOperatorMixin
+from ....dataframe.core import SERIES_TYPE
+from ....dataframe.operators import DataFrameOperatorMixin
 from ....dataframe.utils import parse_index
 from ....serialization.serializables.core import Serializable
 from ....serialization.serializables.field import AnyField, DictField, StringField
@@ -31,24 +34,42 @@ class LLM(Serializable):
         pass
 
 
-class LLMOperator(Operator, TileableOperatorMixin):
+class LLMTaskOperator(Operator, DataFrameOperatorMixin):
+    task = AnyField("task", default=None)
     model = AnyField("model", default=None)
-    prompt_template = AnyField("prompt_template", default=None)
     params = DictField("params", default=None)
+    running_options: Dict[str, Any] = DictField("running_options", default=None)
 
     def __init__(self, output_types=None, **kw):
         if output_types is None:
             output_types = [OutputType.dataframe]
         super().__init__(_output_types=output_types, **kw)
 
-    def __call__(self, data):
-        col_names = ["response", "success"]
-        columns = parse_index(pd.Index(col_names), store_data=True)
-        out_dtypes = pd.Series([np.dtype("O"), np.dtype("bool")], index=col_names)
-        return self.new_tileable(
+    def get_output_dtypes(self) -> Dict[str, np.dtype]:
+        raise NotImplementedError
+
+    def __call__(self, data, index=None):
+        outputs = self.get_output_dtypes()
+        col_name = list(outputs.keys())
+        columns = parse_index(pd.Index(col_name), store_data=True)
+        out_dtypes = pd.Series(list(outputs.values()), index=col_name)
+        index_value = index or (
+            parse_index(pd.RangeIndex(-1), data)
+            if isinstance(data, SERIES_TYPE)
+            else data.index_value
+        )
+
+        return self.new_dataframe(
             inputs=[data],
-            dtypes=out_dtypes,
-            shape=(data.shape[0], len(col_names)),
-            index_value=data.index_value,
+            shape=(np.nan, len(col_name)),
+            index_value=index_value,
             columns_value=columns,
+            dtypes=out_dtypes,
         )
+
+
+class LLMTextGenOperator(LLMTaskOperator, TileableOperatorMixin):
+    prompt_template = AnyField("prompt_template", default=None)
+
+    def get_output_dtypes(self) -> Dict[str, np.dtype]:
+        return {"response": np.dtype("O"), "success": np.dtype("bool")}

maxframe/learn/contrib/llm/models/__init__.py

@@ -11,4 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from .dashscope import DashScopeMultiModalLLM, DashScopeTextLLM

maxframe/learn/contrib/llm/models/dashscope.py

@@ -11,12 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from typing import Any, Dict
 
 from ..... import opcodes
 from .....serialization.serializables.core import Serializable
 from .....serialization.serializables.field import StringField
-from ..core import LLMOperator
+from ..core import LLMTextGenOperator
 from ..multi_modal import MultiModalLLM
 from ..text import TextLLM
 
@@ -65,9 +66,9 @@ class DashScopeMultiModalLLM(MultiModalLLM, DashScopeLLMMixin):
         )(data)
 
 
-class DashScopeTextGenerationOperator(LLMOperator):
+class DashScopeTextGenerationOperator(LLMTextGenOperator):
     _op_type_ = opcodes.DASHSCOPE_TEXT_GENERATION
 
 
-class DashScopeMultiModalGenerationOperator(LLMOperator):
+class DashScopeMultiModalGenerationOperator(LLMTextGenOperator):
     _op_type_ = opcodes.DASHSCOPE_MULTI_MODAL_GENERATION

maxframe/learn/contrib/llm/models/managed.py

@@ -0,0 +1,39 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+from ..... import opcodes
+from .....serialization.serializables import StringField
+from ..core import LLMTextGenOperator
+from ..text import TextLLM
+
+
+class ManagedLLMTextGenOperator(LLMTextGenOperator):
+    _op_type_ = opcodes.MANAGED_TEXT_MODAL_GENERATION
+
+    inference_framework: str = StringField("inference_framework", default=None)
+
+
+class ManagedTextLLM(TextLLM):
+    def generate(
+        self,
+        data,
+        prompt_template: List[Dict[str, Any]],
+        params: Dict[str, Any] = None,
+        **kw
+    ):
+        return ManagedLLMTextGenOperator(
+            model=self, prompt_template=prompt_template, params=params, **kw
+        )(data)

maxframe/learn/contrib/llm/multi_modal.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from typing import Any, Dict
 
 from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE

maxframe/learn/contrib/llm/text.py

@@ -11,32 +11,276 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict
 
-from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
-from .core import LLM
+from typing import Any, Dict, List
+
+import numpy as np
+
+from .... import opcodes
+from ....dataframe.core import DataFrame, Series
+from ....serialization.serializables import FieldTypes, ListField, StringField
+from .core import LLM, LLMTaskOperator
+
+
+class TextLLMSummarizeOperator(LLMTaskOperator):
+    _op_type_ = opcodes.LLM_TEXT_SUMMARIZE_TASK
+
+    def get_output_dtypes(self) -> Dict[str, np.dtype]:
+        return {
+            "summary": np.dtype("O"),
+            "success": np.dtype("bool"),
+        }
+
+
+class TextLLMTranslateOperator(LLMTaskOperator):
+    _op_type_ = opcodes.LLM_TEXT_TRANSLATE_TASK
+
+    source_language = StringField("source_language")
+    target_language = StringField("target_language")
+
+    def get_output_dtypes(self) -> Dict[str, np.dtype]:
+        return {
+            "target": np.dtype("O"),
+            "success": np.dtype("bool"),
+        }
+
+
+class TextLLMClassifyOperator(LLMTaskOperator):
+    _op_type_ = opcodes.LLM_TEXT_CLASSIFY_TASK
+
+    labels = ListField("labels")
+    description = StringField("description", default=None)
+    examples = ListField("examples", FieldTypes.dict, default=None)
+
+    def get_output_dtypes(self) -> Dict[str, np.dtype]:
+        return {
+            "label": np.dtype("O"),
+            "reason": np.dtype("O"),
+            "success": np.dtype("bool"),
+        }
 
 
 class TextLLM(LLM):
     def generate(
         self,
         data,
-        prompt_template: Dict[str, Any],
+        prompt_template: List[Dict[str, str]],
         params: Dict[str, Any] = None,
     ):
         raise NotImplementedError
 
+    def summarize(self, series, index=None, **kw):
+        return TextLLMSummarizeOperator(model=self, task="summarize", **kw)(
+            series, index
+        )
+
+    def translate(
+        self,
+        series,
+        target_language: str,
+        source_language: str = None,
+        index=None,
+        **kw
+    ):
+        return TextLLMTranslateOperator(
+            model=self,
+            task="translate",
+            source_language=source_language,
+            target_language=target_language,
+            **kw
+        )(series, index)
+
+    def classify(
+        self,
+        series,
+        labels: List[str],
+        description=None,
+        examples=None,
+        index=None,
+        **kw
+    ):
+        return TextLLMClassifyOperator(
+            model=self,
+            labels=labels,
+            task="classify",
+            description=description,
+            examples=examples,
+            **kw
+        )(series, index)
+
 
 def generate(
     data,
     model: TextLLM,
-    prompt_template: Dict[str, Any],
+    prompt_template: List[Dict[str, Any]],
     params: Dict[str, Any] = None,
 ):
-    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+    """
+    Generate text using a text language model based on given data and prompt template.
+
+    Parameters
+    ----------
+    data : DataFrame or Series
+        Input data used for generation. Can be maxframe DataFrame, Series that contain text to be processed.
+    model : TextLLM
+        Language model instance used for text generation.
+    prompt_template : List[Dict[str, str]]
+        Dictionary containing the conversation messages template. Use ``{col_name}`` as a placeholder to reference
+        column data from input data.
+
+        Usually in format of [{"role": "user", "content": "{query}"}], same with openai api schema.
+    params : Dict[str, Any], optional
+        Additional parameters for generation configuration, by default None.
+        Can include settings like temperature, max_tokens, etc.
+
+    Returns
+    -------
+    DataFrame
+        Generated text raw response and success status. If the success is False, the generated text will return the
+        error message.
+
+    Examples
+    --------
+    >>> from maxframe.learn.contrib.llm.models.managed import ManagedTextLLM
+    >>> import maxframe.dataframe as md
+    >>>
+    >>> # Initialize the model
+    >>> llm = ManagedTextLLM(name="Qwen2.5-1.5B")
+    >>>
+    >>> # Prepare prompt template
+    >>> messages = [
+    ...     {
+    ...         "role": "user",
+    ...         "content": "{query}",
+    ...     },
+    ... ]
+    >>>
+    >>> # Create sample data
+    >>> df = md.DataFrame({"query": ["What is machine learning?"]})
+    >>>
+    >>> # Generate response
+    >>> result = generate(df, llm, prompt_template=messages)
+    >>> result.execute()
+    """
+    if not isinstance(data, DataFrame) and not isinstance(data, Series):
         raise ValueError("data must be a maxframe dataframe or series object")
     if not isinstance(model, TextLLM):
-        raise ValueError("model must be a TextLLM object")
+        raise TypeError("model must be a TextLLM object")
     params = params if params is not None else dict()
     model.validate_params(params)
-    return model.generate(data, prompt_template, params)
+    return model.generate(data, prompt_template=prompt_template, params=params)
+
+
+def summary(series, model: TextLLM, index=None):
+    """
+    Generate summaries for text content in a series using a language model.
+
+    Parameters
+    ----------
+    series : pandas.Series
+        A maxframe Series containing text data to be summarized.
+        Each element should be a text string.
+    model : TextLLM
+        Language model instance used for text summarization.
+    index : array-like, optional
+        Index for the output series, by default None, will generate new index.
+
+    Returns
+    -------
+    maxframe.Series
+        A pandas Series containing the generated summaries and success status.
+    """
+    if not isinstance(series, Series):
+        raise ValueError("series must be a maxframe series object")
+
+    if series.dtype != np.str_:
+        raise ValueError("summary input must be a string series")
+
+    return model.summarize(series, index=index)
+
+
+def translate(
+    series, model: TextLLM, source_language: str, target_language: str, index=None
+):
+    """
+    Translate text content in a series using a language model from source language to target language.
+
+    Parameters
+    ----------
+    series : pandas.Series
+        A maxframe Series containing text data to be translate.
+        Each element should be a text string.
+    model : TextLLM
+        Language model instance used for text summarization.
+    source_language : str
+        Source language of the text.
+    target_language : str
+        Target language of the text.
+    index : array-like, optional
+        Index for the output series, by default None, will generate new index.
+
+    Returns
+    -------
+    maxframe.Series
+        A pandas Series containing the generated translation and success status.
+    """
+    if not isinstance(series, Series):
+        raise ValueError("series must be a maxframe series object")
+    if series.dtype != np.str_:
+        raise ValueError("translate input must be a string series")
+    return model.translate(
+        series,
+        source_language=source_language,
+        target_language=target_language,
+        index=index,
+    )
+
+
+def classify(
+    series,
+    model: TextLLM,
+    labels: List[str],
+    description: str = None,
+    examples: List[Dict[str, str]] = None,
+    index=None,
+):
+    """
+    Classify text content in a series with given labels.
+
+    Parameters
+    ----------
+    series : pandas.Series
+        A maxframe Series containing text data to be classified.
+        Each element should be a text string.
+    model : TextLLM
+        Language model instance used for text summarization.
+    labels : List[str]
+        List of labels to classify the text.
+    description : str
+        Description of the classification task.
+    examples : List[Dict[str, Dict[str, str]]]
+        Examples of the classification task, like [{ "text": "text...", "label":"A", reason : "reason..."}], help
+        LLM to better understand your rules.
+    index : array-like, optional
+        Index for the output series, by default None, will generate new index.
+
+    Returns
+    -------
+    maxframe.Series
+        A pandas Series containing the generated classification results and success status.
+    """
+    if not isinstance(series, Series):
+        raise ValueError("series must be a maxframe series object")
+
+    if series.dtype != np.str_:
+        raise ValueError("classify input must be a string series")
+
+    if not isinstance(labels, list):
+        raise TypeError("labels must be a list")
+
+    if not labels:
+        raise ValueError("labels must not be empty")
+
+    return model.classify(
+        series, labels=labels, description=description, examples=examples, index=index
+    )

maxframe/learn/contrib/models.py

@@ -0,0 +1,77 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, OutputType
+from ...core.operator import ObjectOperator, ObjectOperatorMixin
+from ...serialization.serializables import (
+    AnyField,
+    DictField,
+    FunctionField,
+    TupleField,
+)
+from ...utils import find_objects, replace_objects
+
+
+class ModelDataSource(ObjectOperator, ObjectOperatorMixin):
+    _op_type_ = opcodes.MODEL_DATA_SOURCE
+
+    data = AnyField("data")
+
+    def __call__(self, model_cls):
+        self._output_types = [OutputType.object]
+        return self.new_tileable(None, object_class=model_cls)
+
+
+class ModelApplyChunk(ObjectOperator, ObjectOperatorMixin):
+    _op_module_ = "maxframe.learn.contrib.models"
+    _op_type_ = opcodes.APPLY_CHUNK
+
+    func = FunctionField("func")
+    args = TupleField("args")
+    kwargs = DictField("kwargs")
+
+    def __init__(self, output_types=None, **kwargs):
+        if not isinstance(output_types, (tuple, list)):
+            output_types = [output_types]
+        self._output_types = list(output_types)
+        super().__init__(**kwargs)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        old_inputs = find_objects(self.args, ENTITY_TYPE) + find_objects(
+            self.kwargs, ENTITY_TYPE
+        )
+        mapping = {o: n for o, n in zip(old_inputs, self._inputs[1:])}
+        self.args = replace_objects(self.args, mapping)
+        self.kwargs = replace_objects(self.kwargs, mapping)
+
+    @property
+    def output_limit(self) -> int:
+        return len(self._output_types)
+
+    def __call__(self, t, output_kws, args=None, **kwargs):
+        self.args = args or ()
+        self.kwargs = kwargs
+        inputs = (
+            [t]
+            + find_objects(self.args, ENTITY_TYPE)
+            + find_objects(self.kwargs, ENTITY_TYPE)
+        )
+        return self.new_tileables(inputs, kws=output_kws)
+
+
+def to_remote_model(model, model_cls):
+    op = ModelDataSource(data=model)
+    return op(model_cls)

maxframe/learn/contrib/utils.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import sys
 
 

maxframe/learn/contrib/xgboost/__init__.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ..utils import config_mod_getattr as _config_mod_getattr
+from .core import Booster
 from .dmatrix import DMatrix
 from .predict import predict
 from .train import train
 
+# isort: off
+from ..utils import config_mod_getattr as _config_mod_getattr
+
 _config_mod_getattr(
     {
         "XGBClassifier": ".classifier.XGBClassifier",
@@ -24,3 +27,7 @@ _config_mod_getattr(
     },
     globals(),
 )
+
+del _config_mod_getattr
+
+__all__ = ["Booster", "DMatrix", "XGBClassifier", "XGBRegressor", "predict", "train"]

maxframe/learn/contrib/xgboost/classifier.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Union
+
 import numpy as np
 
-from ....tensor import argmax, transpose
+from .... import tensor as mt
 from ....tensor.merge.vstack import _vstack
 from ..utils import make_import_error_func
 from .core import XGBScikitLearnBase, xgboost
@@ -33,6 +35,14 @@ else:
         Implementation of the scikit-learn API for XGBoost classification.
         """
 
+        def __init__(
+            self,
+            xgb_model: Union[xgboost.XGBClassifier, xgboost.Booster] = None,
+            **kwargs,
+        ):
+            super().__init__(**kwargs)
+            self._set_model(xgb_model)
+
         def fit(
             self,
             X,
@@ -46,7 +56,7 @@ else:
             **kw,
         ):
             session = kw.pop("session", None)
-            run_kwargs = kw.pop("run_kwargs", dict())
+            run_kwargs = kw.pop("run_kwargs", None) or dict()
             dtrain, evals = wrap_evaluation_matrices(
                 None,
                 X,
@@ -58,6 +68,7 @@ else:
                 base_margin_eval_set,
             )
             params = self.get_xgb_params()
+            self._n_features_in = X.shape[1]
             self.n_classes_ = num_class or 1
             if self.n_classes_ > 2:
                 params["objective"] = "multi:softprob"
@@ -81,7 +92,7 @@ else:
         def predict(self, data, **kw):
             prob = self.predict_proba(data, flag=True, **kw)
             if prob.ndim > 1:
-                prediction = argmax(prob, axis=1)
+                prediction = mt.argmax(prob, axis=1)
             else:
                 prediction = (prob > 0.5).astype(np.int64)
             return prediction
@@ -103,7 +114,7 @@ else:
             # binary logistic function
             classone_probs = prediction
             classzero_probs = 1.0 - classone_probs
-            return transpose(_vstack((classzero_probs, classone_probs)))
+            return mt.transpose(_vstack((classzero_probs, classone_probs)))
 
         @property
         def classes_(self) -> np.ndarray: