maxframe 1.0.0rc2__cp310-cp310-macosx_10_9_universal2.whl → 1.0.0rc4__cp310-cp310-macosx_10_9_universal2.whl

maxframe/learn/contrib/xgboost/predict.py

@@ -12,18 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pickle
 
 import numpy as np
-import pandas as pd
 
 from .... import opcodes
 from ....core.entity.output_types import OutputType
 from ....core.operator.base import Operator
 from ....core.operator.core import TileableOperatorMixin
-from ....dataframe.utils import parse_index
-from ....serialization.serializables import BoolField, BytesField, KeyField, TupleField
-from ....tensor.core import TENSOR_TYPE, TensorOrder
+from ....serialization.serializables import (
+    BoolField,
+    KeyField,
+    ReferenceField,
+    TupleField,
+)
+from ....tensor.core import TensorOrder
+from .core import BoosterData
 from .dmatrix import check_data
 
 
@@ -32,9 +35,7 @@ class XGBPredict(Operator, TileableOperatorMixin):
     output_dtype = np.dtype(np.float32)
 
     data = KeyField("data", default=None)
-    model = BytesField(
-        "model", on_serialize=pickle.dumps, on_deserialize=pickle.loads, default=None
-    )
+    model = ReferenceField("model", reference_type=BoosterData, default=None)
     pred_leaf = BoolField("pred_leaf", default=False)
     pred_contribs = BoolField("pred_contribs", default=False)
     approx_contribs = BoolField("approx_contribs", default=False)
@@ -62,35 +63,12 @@ class XGBPredict(Operator, TileableOperatorMixin):
         else:
             shape = (self.data.shape[0],)
         inputs = [self.data, self.model]
-        if self.output_types[0] == OutputType.tensor:
-            # tensor
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                dtype=self.output_dtype,
-                order=TensorOrder.C_ORDER,
-            )
-        elif self.output_types[0] == OutputType.dataframe:
-            # dataframe
-            dtypes = pd.DataFrame(
-                np.random.rand(0, num_class), dtype=self.output_dtype
-            ).dtypes
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                dtypes=dtypes,
-                columns_value=parse_index(dtypes.index),
-                index_value=self.data.index_value,
-            )
-        else:
-            # series
-            return self.new_tileable(
-                inputs,
-                shape=shape,
-                index_value=self.data.index_value,
-                name="predictions",
-                dtype=self.output_dtype,
-            )
+        return self.new_tileable(
+            inputs,
+            shape=shape,
+            dtype=self.output_dtype,
+            order=TensorOrder.C_ORDER,
+        )
 
 
 def predict(
@@ -107,16 +85,21 @@ def predict(
     strict_shape=False,
     flag=False,
 ):
+    """
+    Using MaxFrame XGBoost model to predict data.
+
+    Parameters
+    ----------
+    Parameters are the same as `xgboost.train`. The predict() is lazy-execution mode.
+
+    Returns
+    -------
+    results: Booster
+    """
     data = check_data(data)
     # TODO: check model datatype
 
-    num_class = getattr(model.op, "num_class", None)
-    if isinstance(data, TENSOR_TYPE):
-        output_types = [OutputType.tensor]
-    elif num_class is not None:
-        output_types = [OutputType.dataframe]
-    else:
-        output_types = [OutputType.series]
+    output_types = [OutputType.tensor]
 
     iteration_range = iteration_range or (0, 0)
 

maxframe/learn/contrib/xgboost/regressor.py

@@ -41,11 +41,6 @@ else:
         ):
             session = kw.pop("session", None)
             run_kwargs = kw.pop("run_kwargs", dict())
-            if kw:
-                raise TypeError(
-                    f"fit got an unexpected keyword argument '{next(iter(kw))}'"
-                )
-
             dtrain, evals = wrap_evaluation_matrices(
                 None,
                 X,
@@ -57,6 +52,8 @@ else:
                 base_margin_eval_set,
             )
             params = self.get_xgb_params()
+            if not params.get("objective"):
+                params["objective"] = "reg:squarederror"
             self.evals_result_ = dict()
             result = train(
                 params,
@@ -71,8 +68,4 @@ else:
             return self
 
         def predict(self, data, **kw):
-            session = kw.pop("session", None)
-            run_kwargs = kw.pop("run_kwargs", None)
-            return predict(
-                self.get_booster(), data, session=session, run_kwargs=run_kwargs, **kw
-            )
+            return predict(self.get_booster(), data, **kw)

maxframe/learn/contrib/xgboost/train.py

@@ -29,6 +29,7 @@ from ....serialization.serializables import (
     KeyField,
     ListField,
 )
+from .core import Booster
 from .dmatrix import ToDMatrix, to_dmatrix
 
 logger = logging.getLogger(__name__)
@@ -59,49 +60,59 @@ class XGBTrain(Operator, TileableOperatorMixin):
     num_boost_round = Int64Field("num_boost_round", default=10)
     num_class = Int64Field("num_class", default=None)
 
-    # Store evals_result in local to store the remote evals_result
-    evals_result: dict = None
-
     def __init__(self, gpu=None, **kw):
         super().__init__(gpu=gpu, **kw)
         if self.output_types is None:
             self.output_types = [OutputType.object]
+        if self.has_evals_result:
+            self.output_types.append(OutputType.object)
 
     def _set_inputs(self, inputs):
         super()._set_inputs(inputs)
         self.dtrain = self._inputs[0]
         rest = self._inputs[1:]
-        if self.evals is not None:
+        if self.has_evals_result:
             evals_dict = OrderedDict(self.evals)
             new_evals_dict = OrderedDict()
             for new_key, val in zip(rest, evals_dict.values()):
                 new_evals_dict[new_key] = val
             self.evals = list(new_evals_dict.items())
 
-    def __call__(self):
+    def __call__(self, evals_result):
         inputs = [self.dtrain]
-        if self.evals is not None:
+        if self.has_evals_result:
             inputs.extend(e[0] for e in self.evals)
-        return self.new_tileable(inputs)
+        return self.new_tileables(
+            inputs, object_class=Booster, evals_result=evals_result
+        )[0]
+
+    @property
+    def output_limit(self):
+        return 2 if self.has_evals_result else 1
+
+    @property
+    def has_evals_result(self) -> bool:
+        return self.evals
 
 
 def train(params, dtrain, evals=None, evals_result=None, num_class=None, **kwargs):
     """
-    Train XGBoost model in Mars manner.
+    Train XGBoost model in MaxFrame manner.
 
     Parameters
     ----------
-    Parameters are the same as `xgboost.train`.
+    Parameters are the same as `xgboost.train`. Note that train is an eager-execution
+    API if evals is passed, thus the call will be blocked until training finished.
 
     Returns
     -------
     results: Booster
     """
 
-    evals_result = evals_result or dict()
-    evals = None or ()
-
+    evals_result = evals_result if evals_result is not None else dict()
     processed_evals = []
+    session = kwargs.pop("session", None)
+    run_kwargs = kwargs.pop("run_kwargs", dict())
     if evals:
         for eval_dmatrix, name in evals:
             if not isinstance(name, str):
@@ -110,12 +121,12 @@ def train(params, dtrain, evals=None, evals_result=None, num_class=None, **kwarg
                 processed_evals.append((eval_dmatrix, name))
             else:
                 processed_evals.append((to_dmatrix(eval_dmatrix), name))
-
-    return XGBTrain(
+    data = XGBTrain(
         params=params,
         dtrain=dtrain,
         evals=processed_evals,
         evals_result=evals_result,
         num_class=num_class,
-        **kwargs
-    )()
+        **kwargs,
+    )(evals_result)
+    return data.execute(session=session, **run_kwargs) if evals else data

maxframe/{core/operator/fuse.py → learn/core.py}

@@ -12,18 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ... import opcodes
-from ...serialization.serializables import ReferenceField
-from ..graph import ChunkGraph
-from .base import Operator
+from ..core.entity.objects import Object, ObjectData
 
 
-class Fuse(Operator):
-    __slots__ = ("_fuse_graph",)
-    _op_type_ = opcodes.FUSE
+class ModelData(ObjectData):
+    pass
 
-    fuse_graph = ReferenceField("fuse_graph", ChunkGraph)
 
+class Model(Object):
+    pass
 
-class FuseChunkMixin:
-    __slots__ = ()
+
+MODEL_TYPE = (Model, ModelData)

maxframe/opcodes.py

@@ -532,6 +532,8 @@ STATSMODELS_TRAIN = 3012
 STATSMODELS_PREDICT = 3013
 
 # learn
+CONNECTED_COMPONENTS = 3100
+
 # checks
 CHECK_NON_NEGATIVE = 3300
 # classifier check targets
@@ -566,6 +568,7 @@ CHOLESKY_FUSE = 999988
 
 # MaxFrame-dedicated functions
 DATAFRAME_RESHUFFLE = 10001
+FLATMAP = 10002
 
 # MaxFrame internal operators
 DATAFRAME_PROJECTION_SAME_INDEX_MERGE = 100001

maxframe/protocol.py

@@ -15,7 +15,7 @@
 import base64
 import enum
 import uuid
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
+from typing import Any, Dict, Generic, List, Optional, Type, TypeVar
 
 import pandas as pd
 
@@ -38,7 +38,6 @@ from .serialization.serializables import (
     Serializable,
     SeriesField,
     StringField,
-    TupleField,
 )
 
 pickling_support.install()
@@ -92,19 +91,6 @@ class DataSerializeType(enum.Enum):
     PICKLE = 0
 
 
-class VolumeDataMeta(Serializable):
-    output_type: OutputType = EnumField(
-        "output_type", OutputType, FieldTypes.int8, default=None
-    )
-    serial_type: DataSerializeType = EnumField(
-        "serial_type", DataSerializeType, FieldTypes.int8, default=None
-    )
-    shape: Tuple[int, ...] = TupleField("shape", FieldTypes.int64, default=None)
-    nsplits: Tuple[Tuple[int, ...], ...] = TupleField(
-        "nsplits", FieldTypes.tuple(FieldTypes.tuple(FieldTypes.int64)), default=None
-    )
-
-
 _result_type_to_info_cls: Dict[ResultType, Type["ResultInfo"]] = dict()
 
 
@@ -389,6 +375,11 @@ class ExecuteDagRequest(Serializable):
         value_type=FieldTypes.reference,
         default=None,
     )
+    new_settings: Dict[str, Any] = DictField(
+        "new_settings",
+        key_type=FieldTypes.string,
+        default=None,
+    )
 
 
 class SubDagSubmitInstanceInfo(JsonSerializable):
@@ -525,7 +516,7 @@ class DataFrameTableMeta(JsonSerializable):
         return True
 
     def to_json(self) -> dict:
-        b64_pk = lambda x: base64.b64encode(pickle.dumps(x))
+        b64_pk = lambda x: base64.b64encode(pickle.dumps(x)).decode()
         ret = {
             "table_name": self.table_name,
             "type": self.type.value,

maxframe/remote/core.py

@@ -15,7 +15,7 @@
 from functools import partial
 
 from .. import opcodes
-from ..core import ENTITY_TYPE, ChunkData
+from ..core import ENTITY_TYPE
 from ..core.operator import ObjectOperator, ObjectOperatorMixin
 from ..dataframe.core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
 from ..serialization.serializables import (
@@ -26,7 +26,7 @@ from ..serialization.serializables import (
     ListField,
 )
 from ..tensor.core import TENSOR_TYPE
-from ..utils import build_fetch_tileable, find_objects, replace_objects
+from ..utils import find_objects, replace_objects
 
 
 class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
@@ -63,12 +63,8 @@ class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
         if raw_inputs is not None:
             for raw_inp in raw_inputs:
                 if self._no_prepare(raw_inp):
-                    if not isinstance(self._inputs[0], ChunkData):
-                        # not in tile, set_inputs from tileable
-                        mapping[raw_inp] = next(function_inputs)
-                    else:
-                        # in tile, set_inputs from chunk
-                        mapping[raw_inp] = build_fetch_tileable(raw_inp)
+                    # not in tile, set_inputs from tileable
+                    mapping[raw_inp] = next(function_inputs)
                 else:
                     mapping[raw_inp] = next(function_inputs)
         self.function_args = replace_objects(self.function_args, mapping)

maxframe/serialization/__init__.py

@@ -17,6 +17,7 @@ from .core import (
     PickleContainer,
     Serializer,
     deserialize,
+    load_type,
     pickle_buffers,
     serialize,
     serialize_with_spawn,

maxframe/session.py

@@ -150,6 +150,10 @@ class AbstractSession(ABC):
     def session_id(self):
         return self._session_id
 
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
     def __eq__(self, other):
         return (
             isinstance(other, AbstractSession)
@@ -1283,9 +1287,12 @@ def get_default_or_create(**kwargs):
         if session is None:
             # no session attached, try to create one
             warnings.warn(warning_msg)
-            session = new_session(
-                ODPS.from_global() or ODPS.from_environments(), **kwargs
+            odps_entry = (
+                kwargs.pop("odps_entry", None)
+                or ODPS.from_global()
+                or ODPS.from_environments()
             )
+            session = new_session(odps_entry=odps_entry, **kwargs)
             session.as_default()
     if isinstance(session, IsolatedAsyncSession):
         session = SyncSession.from_isolated_session(session)

maxframe/tensor/__init__.py

@@ -114,7 +114,6 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,
@@ -143,7 +142,16 @@ from .indexing import (
     take,
     unravel_index,
 )
-from .merge import stack
+from .merge import concatenate, stack, vstack
+from .misc import (
+    atleast_1d,
+    atleast_2d,
+    atleast_3d,
+    broadcast_to,
+    transpose,
+    unique,
+    where,
+)
 from .rechunk import rechunk
 from .reduction import (
     all,

maxframe/tensor/arithmetic/isclose.py

@@ -23,6 +23,7 @@ from .core import TensorBinOp
 
 class TensorIsclose(TensorBinOp):
     _op_type_ = opcodes.ISCLOSE
+    _func_name = "isclose"
 
     rtol = Float64Field("rtol", default=None)
     atol = Float64Field("atol", default=None)

maxframe/tensor/arithmetic/tests/test_arithmetic.py

@@ -17,26 +17,13 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.arithmetic.core import TensorBinOp, TensorUnaryOp
+from maxframe.utils import collect_leaf_operators
+
 from ....core import enter_mode
 from ...core import SparseTensor, Tensor
 from ...datasource import array, empty, ones, tensor
-from .. import (
-    TensorAdd,
-    TensorGreaterThan,
-    TensorIsclose,
-    TensorLog,
-    TensorSubtract,
-    add,
-    around,
-    cos,
-    frexp,
-    isclose,
-    isfinite,
-    log,
-    negative,
-    subtract,
-    truediv,
-)
+from .. import *  # noqa: F401
 
 
 def test_add():
@@ -412,3 +399,20 @@ def test_build_mode():
 
     with enter_mode(build=True):
         assert t1 != 2
+
+
+def test_unary_op_func_name():
+    # make sure all the unary op has defined the func name.
+
+    results = collect_leaf_operators(TensorUnaryOp)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")
+
+
+def test_binary_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorBinOp)
+    for op_type in results:
+        if op_type not in (TensorSetImag, TensorSetReal):
+            assert hasattr(op_type, "_func_name")