maxframe 0.1.0b4__cp310-cp310-macosx_10_9_universal2.whl → 1.0.0__cp310-cp310-macosx_10_9_universal2.whl

maxframe/learn/contrib/xgboost/tests/test_core.py

@@ -0,0 +1,43 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+try:
+    import xgboost
+except ImportError:
+    xgboost = None
+
+
+from ..... import tensor as mt
+
+if xgboost:
+    from ..core import wrap_evaluation_matrices
+
+
+@pytest.mark.skipif(xgboost is None, reason="XGBoost not installed")
+def test_wrap_evaluation_matrices():
+    X = mt.random.rand(100, 3)
+    y = mt.random.randint(3, size=(100,))
+
+    eval_set = [(mt.random.rand(10, 3), mt.random.randint(3, size=10))]
+    with pytest.raises(ValueError):
+        # sample_weight_eval_set size wrong
+        wrap_evaluation_matrices(0.0, X, y, None, None, eval_set, [], None)
+
+    with pytest.raises(ValueError):
+        wrap_evaluation_matrices(0.0, X, y, None, None, None, eval_set, None)
+
+    evals = wrap_evaluation_matrices(0.0, X, y, None, None, eval_set, None, None)[1]
+    assert len(evals) > 0

maxframe/learn/contrib/xgboost/train.py

@@ -0,0 +1,132 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from collections import OrderedDict
+
+from .... import opcodes
+from ....core import OutputType
+from ....core.operator.base import Operator
+from ....core.operator.core import TileableOperatorMixin
+from ....serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    FieldTypes,
+    FunctionField,
+    Int64Field,
+    KeyField,
+    ListField,
+)
+from .core import Booster
+from .dmatrix import ToDMatrix, to_dmatrix
+
+logger = logging.getLogger(__name__)
+
+
+def _on_serialize_evals(evals_val):
+    if evals_val is None:
+        return None
+    return [list(x) for x in evals_val]
+
+
+class XGBTrain(Operator, TileableOperatorMixin):
+    _op_type_ = opcodes.XGBOOST_TRAIN
+
+    params = DictField("params", key_type=FieldTypes.string, default=None)
+    dtrain = KeyField("dtrain", default=None)
+    evals = ListField("evals", on_serialize=_on_serialize_evals, default=None)
+    obj = FunctionField("obj", default=None)
+    feval = FunctionField("obj", default=None)
+    maximize = BoolField("maximize", default=None)
+    early_stopping_rounds = Int64Field("early_stopping_rounds", default=None)
+    verbose_eval = AnyField("verbose_eval", default=None)
+    xgb_model = AnyField("xgb_model", default=None)
+    callbacks = ListField(
+        "callbacks", field_type=FunctionField.field_type, default=None
+    )
+    custom_metric = FunctionField("custom_metric", default=None)
+    num_boost_round = Int64Field("num_boost_round", default=10)
+    num_class = Int64Field("num_class", default=None)
+
+    def __init__(self, gpu=None, **kw):
+        super().__init__(gpu=gpu, **kw)
+        if self.output_types is None:
+            self.output_types = [OutputType.object]
+        if self.has_evals_result:
+            self.output_types.append(OutputType.object)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        self.dtrain = self._inputs[0]
+        rest = self._inputs[1:]
+        if self.has_evals_result:
+            evals_dict = OrderedDict(self.evals)
+            new_evals_dict = OrderedDict()
+            for new_key, val in zip(rest, evals_dict.values()):
+                new_evals_dict[new_key] = val
+            self.evals = list(new_evals_dict.items())
+
+    def __call__(self, evals_result):
+        inputs = [self.dtrain]
+        if self.has_evals_result:
+            inputs.extend(e[0] for e in self.evals)
+        return self.new_tileables(
+            inputs, object_class=Booster, evals_result=evals_result
+        )[0]
+
+    @property
+    def output_limit(self):
+        return 2 if self.has_evals_result else 1
+
+    @property
+    def has_evals_result(self) -> bool:
+        return self.evals
+
+
+def train(params, dtrain, evals=None, evals_result=None, num_class=None, **kwargs):
+    """
+    Train XGBoost model in MaxFrame manner.
+
+    Parameters
+    ----------
+    Parameters are the same as `xgboost.train`. Note that train is an eager-execution
+    API if evals is passed, thus the call will be blocked until training finished.
+
+    Returns
+    -------
+    results: Booster
+    """
+
+    evals_result = evals_result if evals_result is not None else dict()
+    processed_evals = []
+    session = kwargs.pop("session", None)
+    run_kwargs = kwargs.pop("run_kwargs", dict())
+    if evals:
+        for eval_dmatrix, name in evals:
+            if not isinstance(name, str):
+                raise TypeError("evals must a list of pairs (DMatrix, string)")
+            if hasattr(eval_dmatrix, "op") and isinstance(eval_dmatrix.op, ToDMatrix):
+                processed_evals.append((eval_dmatrix, name))
+            else:
+                processed_evals.append((to_dmatrix(eval_dmatrix), name))
+    data = XGBTrain(
+        params=params,
+        dtrain=dtrain,
+        evals=processed_evals,
+        evals_result=evals_result,
+        num_class=num_class,
+        **kwargs,
+    )(evals_result)
+    return data.execute(session=session, **run_kwargs) if evals else data

maxframe/{core/operator/fuse.py → learn/core.py}

@@ -12,18 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ... import opcodes
-from ...serialization.serializables import ReferenceField
-from ..graph import ChunkGraph
-from .base import Operator
+from ..core.entity.objects import Object, ObjectData
 
 
-class Fuse(Operator):
-    __slots__ = ("_fuse_graph",)
-    _op_type_ = opcodes.FUSE
+class ModelData(ObjectData):
+    pass
 
-    fuse_graph = ReferenceField("fuse_graph", ChunkGraph)
 
+class Model(Object):
+    pass
 
-class FuseChunkMixin:
-    __slots__ = ()
+
+MODEL_TYPE = (Model, ModelData)

maxframe/learn/utils/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import convert_to_tensor_or_dataframe

maxframe/learn/utils/core.py

@@ -0,0 +1,29 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ...dataframe import DataFrame, Series
+from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from ...tensor import tensor as astensor
+
+
+def convert_to_tensor_or_dataframe(item):
+    if isinstance(item, (DATAFRAME_TYPE, pd.DataFrame)):
+        item = DataFrame(item)
+    elif isinstance(item, (SERIES_TYPE, pd.Series)):
+        item = Series(item)
+    else:
+        item = astensor(item)
+    return item

maxframe/lib/mmh3.pyi

@@ -0,0 +1,43 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple
+
+def hash(key, seed=0, signed=True) -> int:
+    """
+    Return a 32 bit integer.
+    """
+
+def hash_from_buffer(key, seed=0, signed=True) -> int:
+    """
+    Return a 32 bit integer. Designed for large memory-views such as numpy arrays.
+    """
+
+def hash64(key, seed=0, x64arch=True, signed=True) -> Tuple[int, int]:
+    """
+    Return a tuple of two 64 bit integers for a string. Optimized for
+    the x64 bit architecture when x64arch=True, otherwise for x86.
+    """
+
+def hash128(key, seed=0, x64arch=True, signed=False) -> int:
+    """
+    Return a 128 bit long integer. Optimized for the x64 bit architecture
+    when x64arch=True, otherwise for x86.
+    """
+
+def hash_bytes(key, seed=0, x64arch=True) -> bytes:
+    """
+    Return a 128 bit hash value as bytes for a string. Optimized for the
+    x64 bit architecture when x64arch=True, otherwise for the x86.
+    """

maxframe/lib/sparse/tests/test_sparse.py

@@ -55,13 +55,13 @@ def test_sparse_creation():
     s = SparseNDArray(s1_data)
     assert s.ndim == 2
     assert isinstance(s, SparseMatrix)
-    assert_array_equal(s.toarray(), s1_data.A)
-    assert_array_equal(s.todense(), s1_data.A)
+    assert_array_equal(s.toarray(), s1_data.toarray())
+    assert_array_equal(s.todense(), s1_data.toarray())
 
     ss = pickle.loads(pickle.dumps(s))
     assert s == ss
-    assert_array_equal(ss.toarray(), s1_data.A)
-    assert_array_equal(ss.todense(), s1_data.A)
+    assert_array_equal(ss.toarray(), s1_data.toarray())
+    assert_array_equal(ss.todense(), s1_data.toarray())
 
     v = SparseNDArray(v1, shape=(3,))
     assert s.ndim
@@ -331,12 +331,12 @@ def test_sparse_dot():
 
     assert_array_equal(mls.dot(s1, v1_s), s1.dot(v1_data))
     assert_array_equal(mls.dot(s2, v1_s), s2.dot(v1_data))
-    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.A))
-    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.A))
+    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.toarray()))
+    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s), v1_data.dot(v1_data), almost=True)
     assert_array_equal(mls.dot(v2_s, v2_s), v2_data.dot(v2_data), almost=True)
 
-    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.A))
+    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s, sparse=False), v1_data.dot(v1_data))
 
 
@@ -390,7 +390,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -399,7 +399,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -408,7 +408,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3])
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3])
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -417,7 +417,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3], wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3], wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -427,7 +427,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -437,7 +437,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -447,7 +447,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -457,7 +457,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)

maxframe/lib/wrapped_pickle.py

@@ -120,7 +120,8 @@ class _UnpickleSwitch:
             @functools.wraps(func)
             async def wrapped(*args, **kwargs):
                 with _UnpickleSwitch(forbidden=self._forbidden):
-                    return await func(*args, **kwargs)
+                    ret = await func(*args, **kwargs)
+                return ret
 
         else:
 

maxframe/opcodes.py

@@ -386,9 +386,16 @@ DATAFRAME_EVAL = 738
 DUPLICATED = 739
 DELETE = 740
 ALIGN = 741
+CASE_WHEN = 742
+PIVOT = 743
+PIVOT_TABLE = 744
 
 FUSE = 801
 
+# LLM
+DASHSCOPE_TEXT_GENERATION = 810
+DASHSCOPE_MULTI_MODAL_GENERATION = 811
+
 # table like input for tensor
 TABLE_COO = 1003
 # store tensor as coo format
@@ -529,6 +536,8 @@ STATSMODELS_TRAIN = 3012
 STATSMODELS_PREDICT = 3013
 
 # learn
+CONNECTED_COMPONENTS = 3100
+
 # checks
 CHECK_NON_NEGATIVE = 3300
 # classifier check targets
@@ -563,6 +572,8 @@ CHOLESKY_FUSE = 999988
 
 # MaxFrame-dedicated functions
 DATAFRAME_RESHUFFLE = 10001
+FLATMAP = 10002
+FLATJSON = 10003
 
 # MaxFrame internal operators
 DATAFRAME_PROJECTION_SAME_INDEX_MERGE = 100001