maxframe 0.1.0b5__cp38-cp38-macosx_10_9_universal2.whl → 1.0.0__cp38-cp38-macosx_10_9_universal2.whl

maxframe/learn/contrib/graph/tests/test_connected_components.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pytest
+
+from ..... import dataframe as md
+from .....dataframe.core import DataFrameData
+from .....tensor.core import TensorData
+from .. import connected_components
+
+
+@pytest.fixture
+def df1():
+    return md.DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+
+
+@pytest.fixture
+def df2():
+    return md.DataFrame(
+        [[1, "2"], [1, "2"]],
+        columns=["a", "b"],
+    )
+
+
+def test_connected_components(df1, df2):
+    edges, flag = connected_components(df1, "a", "b")
+    assert edges.op.max_iter == 6
+    assert edges.shape == (np.nan, 2)
+    assert isinstance(edges.data, DataFrameData)
+    assert isinstance(flag.data, TensorData)
+    assert flag.shape == ()
+    assert "id" in edges.dtypes and "component" in edges.dtypes
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "x")
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "b", 0)
+
+    with pytest.raises(ValueError):
+        connected_components(df2, "a", "b")

maxframe/learn/contrib/llm/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from . import models, multi_modal, text
+
+del models

maxframe/learn/contrib/llm/core.py

@@ -0,0 +1,54 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+import numpy as np
+import pandas as pd
+
+from ....core.entity.output_types import OutputType
+from ....core.operator.base import Operator
+from ....core.operator.core import TileableOperatorMixin
+from ....dataframe.utils import parse_index
+from ....serialization.serializables.core import Serializable
+from ....serialization.serializables.field import AnyField, DictField, StringField
+
+
+class LLM(Serializable):
+    name = StringField("name", default=None)
+
+    def validate_params(self, params: Dict[str, Any]):
+        pass
+
+
+class LLMOperator(Operator, TileableOperatorMixin):
+    model = AnyField("model", default=None)
+    prompt_template = AnyField("prompt_template", default=None)
+    params = DictField("params", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        if output_types is None:
+            output_types = [OutputType.dataframe]
+        super().__init__(_output_types=output_types, **kw)
+
+    def __call__(self, data):
+        col_names = ["response", "success"]
+        columns = parse_index(pd.Index(col_names), store_data=True)
+        out_dtypes = pd.Series([np.dtype("O"), np.dtype("bool")], index=col_names)
+        return self.new_tileable(
+            inputs=[data],
+            dtypes=out_dtypes,
+            shape=(data.shape[0], len(col_names)),
+            index_value=data.index_value,
+            columns_value=columns,
+        )

maxframe/learn/contrib/llm/models/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .dashscope import DashScopeMultiModalLLM, DashScopeTextLLM

maxframe/learn/contrib/llm/models/dashscope.py

@@ -0,0 +1,73 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ..... import opcodes
+from .....serialization.serializables.core import Serializable
+from .....serialization.serializables.field import StringField
+from ..core import LLMOperator
+from ..multi_modal import MultiModalLLM
+from ..text import TextLLM
+
+
+class DashScopeLLMMixin(Serializable):
+    __slots__ = ()
+
+    _not_supported_params = {"stream", "incremental_output"}
+
+    def validate_params(self, params: Dict[str, Any]):
+        for k in params.keys():
+            if k in self._not_supported_params:
+                raise ValueError(f"{k} is not supported")
+
+
+class DashScopeTextLLM(TextLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        return DashScopeTextGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeMultiModalLLM(MultiModalLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        # TODO add precheck here
+        return DashScopeMultiModalGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeTextGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_TEXT_GENERATION
+
+
+class DashScopeMultiModalGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_MULTI_MODAL_GENERATION

maxframe/learn/contrib/llm/multi_modal.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class MultiModalLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: MultiModalLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, MultiModalLLM):
+        raise ValueError("model must be a MultiModalLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/learn/contrib/llm/text.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class TextLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: TextLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, TextLLM):
+        raise ValueError("model must be a TextLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/learn/contrib/xgboost/classifier.py

@@ -14,7 +14,8 @@
 
 import numpy as np
 
-from ....tensor import argmax
+from ....tensor import argmax, transpose
+from ....tensor.merge.vstack import _vstack
 from ..utils import make_import_error_func
 from .core import XGBScikitLearnBase, xgboost
 
@@ -42,7 +43,10 @@ else:
             sample_weight_eval_set=None,
             base_margin_eval_set=None,
             num_class=None,
+            **kw,
         ):
+            session = kw.pop("session", None)
+            run_kwargs = kw.pop("run_kwargs", dict())
             dtrain, evals = wrap_evaluation_matrices(
                 None,
                 X,
@@ -68,6 +72,8 @@ else:
                 evals=evals,
                 evals_result=self.evals_result_,
                 num_class=num_class,
+                session=session,
+                run_kwargs=run_kwargs,
             )
             self._Booster = result
             return self
@@ -83,4 +89,22 @@ else:
         def predict_proba(self, data, ntree_limit=None, flag=False, **kw):
             if ntree_limit is not None:
                 raise NotImplementedError("ntree_limit is not currently supported")
-            return predict(self.get_booster(), data, flag=flag, **kw)
+            prediction = predict(self.get_booster(), data, flag=flag, **kw)
+            if len(prediction.shape) == 2 and prediction.shape[1] == self.n_classes_:
+                # multi-class
+                return prediction
+            if (
+                len(prediction.shape) == 2
+                and self.n_classes_ == 2
+                and prediction.shape[1] >= self.n_classes_
+            ):
+                # multi-label
+                return prediction
+            # binary logistic function
+            classone_probs = prediction
+            classzero_probs = 1.0 - classone_probs
+            return transpose(_vstack((classzero_probs, classone_probs)))
+
+        @property
+        def classes_(self) -> np.ndarray:
+            return np.arange(self.n_classes_)

maxframe/learn/contrib/xgboost/core.py

@@ -12,15 +12,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 try:
     import xgboost
 except ImportError:
     xgboost = None
 
+from ...core import Model, ModelData
 from .dmatrix import DMatrix
 
+
+class BoosterData(ModelData):
+    __slots__ = ("_evals_result",)
+
+    _evals_result: Dict
+
+    def __init__(self, *args, evals_result=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._evals_result = evals_result if evals_result is not None else dict()
+
+    def execute(self, session=None, **kw):
+        # The evals_result should be fetched when BoosterData.execute() is called.
+        result = super().execute(session=session, **kw)
+        if self.op.has_evals_result and self.key == self.op.outputs[0].key:
+            self._evals_result.update(self.op.outputs[1].fetch(session=session))
+        return result
+
+    def predict(
+        self,
+        data,
+        output_margin=False,
+        pred_leaf=False,
+        pred_contribs=False,
+        approx_contribs=False,
+        pred_interactions=False,
+        validate_features=True,
+        training=False,
+        iteration_range=None,
+        strict_shape=False,
+    ):
+        from .predict import predict
+
+        return predict(
+            self,
+            data,
+            output_margin=output_margin,
+            pred_leaf=pred_leaf,
+            pred_contribs=pred_contribs,
+            approx_contribs=approx_contribs,
+            pred_interactions=pred_interactions,
+            validate_features=validate_features,
+            training=training,
+            iteration_range=iteration_range,
+            strict_shape=strict_shape,
+        )
+
+
+class Booster(Model):
+    pass
+
+
 if not xgboost:
     XGBScikitLearnBase = None
 else:
@@ -40,7 +92,9 @@ else:
             **kw,
         ):
             """
-            Fit the regressor.
+            Fit the regressor. Note that fit() is an eager-execution
+            API. The call will be blocked until training finished.
+
             Parameters
             ----------
             X : array_like
@@ -72,6 +126,37 @@ else:
             """
             raise NotImplementedError
 
+        def evals_result(self, **kw) -> Dict:
+            """Return the evaluation results.
+
+            If **eval_set** is passed to the :py:meth:`fit` function, you can call
+            ``evals_result()`` to get evaluation results for all passed **eval_sets**.  When
+            **eval_metric** is also passed to the :py:meth:`fit` function, the
+            **evals_result** will contain the **eval_metrics** passed to the :py:meth:`fit`
+            function.
+
+            The returned evaluation result is a dictionary:
+
+            .. code-block:: python
+
+                {'validation_0': {'logloss': ['0.604835', '0.531479']},
+                 'validation_1': {'logloss': ['0.41965', '0.17686']}}
+
+            Note that evals_result() will be blocked until the train is finished.
+
+            Returns
+            -------
+            evals_result
+
+            """
+            result = super().evals_result()
+            if not self._Booster.op.has_evals_result or len(result) != 0:
+                return result
+            session = kw.pop("session", None)
+            run_kwargs = kw.pop("run_kwargs", dict())
+            self._Booster.execute(session=session, **run_kwargs)
+            return super().evals_result()
+
     def wrap_evaluation_matrices(
         missing: float,
         X: Any,

maxframe/learn/contrib/xgboost/dmatrix.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-from .... import opcodes as OperandDef
+from .... import opcodes
 from ....core.entity.output_types import get_output_types
 from ....core.operator.base import Operator
 from ....core.operator.core import TileableOperatorMixin
@@ -27,7 +27,7 @@ from ...utils import convert_to_tensor_or_dataframe
 
 
 class ToDMatrix(Operator, TileableOperatorMixin):
-    _op_type_ = OperandDef.TO_DMATRIX
+    _op_type_ = opcodes.TO_DMATRIX
 
     data = KeyField("data", default=None)
     label = KeyField("label", default=None)
@@ -99,10 +99,7 @@ def check_array_like(y: TileableType, name: str) -> TileableType:
     y = convert_to_tensor_or_dataframe(y)
     if isinstance(y, DATAFRAME_TYPE):
         y = y.iloc[:, 0]
-    y = astensor(y)
-    if y.ndim != 1:
-        raise ValueError(f"Expecting 1-d {name}, got: {y.ndim}-d")
-    return y
+    return astensor(y)
 
 
 def to_dmatrix(

maxframe/learn/contrib/xgboost/predict.py

@@ -12,29 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pickle
 
 import numpy as np
-import pandas as pd
 
-from .... import opcodes as OperandDef
+from .... import opcodes
 from ....core.entity.output_types import OutputType
 from ....core.operator.base import Operator
 from ....core.operator.core import TileableOperatorMixin
-from ....dataframe.utils import parse_index
-from ....serialization.serializables import BoolField, BytesField, KeyField, TupleField
-from ....tensor.core import TENSOR_TYPE, TensorOrder
+from ....serialization.serializables import (
+    BoolField,
+    KeyField,
+    ReferenceField,
+    TupleField,
+)
+from ....tensor.core import TensorOrder
+from .core import BoosterData
 from .dmatrix import check_data
 
 
 class XGBPredict(Operator, TileableOperatorMixin):
-    _op_type_ = OperandDef.XGBOOST_PREDICT
+    _op_type_ = opcodes.XGBOOST_PREDICT
     output_dtype = np.dtype(np.float32)
 
     data = KeyField("data", default=None)
-    model = BytesField(
-        "model", on_serialize=pickle.dumps, on_deserialize=pickle.loads, default=None
-    )
+    model = ReferenceField("model", reference_type=BoosterData, default=None)
     pred_leaf = BoolField("pred_leaf", default=False)
     pred_contribs = BoolField("pred_contribs", default=False)
     approx_contribs = BoolField("approx_contribs", default=False)
@@ -62,35 +63,12 @@ class XGBPredict(Operator, TileableOperatorMixin):
         else:
             shape = (self.data.shape[0],)
         inputs = [self.data, self.model]
-        if self.output_types[0] == OutputType.tensor:
-            # tensor
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                dtype=self.output_dtype,
-                order=TensorOrder.C_ORDER,
-            )
-        elif self.output_types[0] == OutputType.dataframe:
-            # dataframe
-            dtypes = pd.DataFrame(
-                np.random.rand(0, num_class), dtype=self.output_dtype
-            ).dtypes
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                dtypes=dtypes,
-                columns_value=parse_index(dtypes.index),
-                index_value=self.data.index_value,
-            )
-        else:
-            # series
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                index_value=self.data.index_value,
-                name="predictions",
-                dtype=self.output_dtype,
-            )
+        return self.new_tileable(
+            inputs,
+            shape=shape,
+            dtype=self.output_dtype,
+            order=TensorOrder.C_ORDER,
+        )
 
 
 def predict(
@@ -107,16 +85,21 @@ def predict(
     strict_shape=False,
     flag=False,
 ):
+    """
+    Using MaxFrame XGBoost model to predict data.
+
+    Parameters
+    ----------
+    Parameters are the same as `xgboost.train`. The predict() is lazy-execution mode.
+
+    Returns
+    -------
+    results: Booster
+    """
     data = check_data(data)
     # TODO: check model datatype
 
-    num_class = getattr(model.op, "num_class", None)
-    if isinstance(data, TENSOR_TYPE):
-        output_types = [OutputType.tensor]
-    elif num_class is not None:
-        output_types = [OutputType.dataframe]
-    else:
-        output_types = [OutputType.series]
+    output_types = [OutputType.tensor]
 
     iteration_range = iteration_range or (0, 0)
 

maxframe/learn/contrib/xgboost/regressor.py

@@ -41,11 +41,6 @@ else:
         ):
             session = kw.pop("session", None)
             run_kwargs = kw.pop("run_kwargs", dict())
-            if kw:
-                raise TypeError(
-                    f"fit got an unexpected keyword argument '{next(iter(kw))}'"
-                )
-
             dtrain, evals = wrap_evaluation_matrices(
                 None,
                 X,
@@ -57,6 +52,8 @@ else:
                 base_margin_eval_set,
             )
             params = self.get_xgb_params()
+            if not params.get("objective"):
+                params["objective"] = "reg:squarederror"
             self.evals_result_ = dict()
             result = train(
                 params,
@@ -71,8 +68,4 @@ else:
             return self
 
         def predict(self, data, **kw):
-            session = kw.pop("session", None)
-            run_kwargs = kw.pop("run_kwargs", None)
-            return predict(
-                self.get_booster(), data, session=session, run_kwargs=run_kwargs, **kw
-            )
+            return predict(self.get_booster(), data, **kw)