maxframe 1.0.0rc1__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0rc3__cp39-cp39-macosx_10_9_universal2.whl

maxframe/learn/contrib/xgboost/train.py

@@ -15,7 +15,7 @@
 import logging
 from collections import OrderedDict
 
-from .... import opcodes as OperandDef
+from .... import opcodes
 from ....core import OutputType
 from ....core.operator.base import Operator
 from ....core.operator.core import TileableOperatorMixin
@@ -29,6 +29,7 @@ from ....serialization.serializables import (
     KeyField,
     ListField,
 )
+from .core import Booster
 from .dmatrix import ToDMatrix, to_dmatrix
 
 logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ def _on_serialize_evals(evals_val):
 
 
 class XGBTrain(Operator, TileableOperatorMixin):
-    _op_type_ = OperandDef.XGBOOST_TRAIN
+    _op_type_ = opcodes.XGBOOST_TRAIN
 
     params = DictField("params", key_type=FieldTypes.string, default=None)
     dtrain = KeyField("dtrain", default=None)
@@ -59,49 +60,59 @@ class XGBTrain(Operator, TileableOperatorMixin):
     num_boost_round = Int64Field("num_boost_round", default=10)
     num_class = Int64Field("num_class", default=None)
 
-    # Store evals_result in local to store the remote evals_result
-    evals_result: dict = None
-
     def __init__(self, gpu=None, **kw):
         super().__init__(gpu=gpu, **kw)
         if self.output_types is None:
             self.output_types = [OutputType.object]
+        if self.has_evals_result:
+            self.output_types.append(OutputType.object)
 
     def _set_inputs(self, inputs):
         super()._set_inputs(inputs)
         self.dtrain = self._inputs[0]
         rest = self._inputs[1:]
-        if self.evals is not None:
+        if self.has_evals_result:
             evals_dict = OrderedDict(self.evals)
             new_evals_dict = OrderedDict()
             for new_key, val in zip(rest, evals_dict.values()):
                 new_evals_dict[new_key] = val
             self.evals = list(new_evals_dict.items())
 
-    def __call__(self):
+    def __call__(self, evals_result):
         inputs = [self.dtrain]
-        if self.evals is not None:
+        if self.has_evals_result:
             inputs.extend(e[0] for e in self.evals)
-        return self.new_tileable(inputs)
+        return self.new_tileables(
+            inputs, object_class=Booster, evals_result=evals_result
+        )[0]
+
+    @property
+    def output_limit(self):
+        return 2 if self.has_evals_result else 1
+
+    @property
+    def has_evals_result(self) -> bool:
+        return self.evals
 
 
 def train(params, dtrain, evals=None, evals_result=None, num_class=None, **kwargs):
     """
-    Train XGBoost model in Mars manner.
+    Train XGBoost model in MaxFrame manner.
 
     Parameters
     ----------
-    Parameters are the same as `xgboost.train`.
+    Parameters are the same as `xgboost.train`. Note that train is an eager-execution
+    API. The call will be blocked until training finished.
 
     Returns
     -------
     results: Booster
     """
 
-    evals_result = evals_result or dict()
-    evals = None or ()
-
+    evals_result = evals_result if evals_result is not None else dict()
     processed_evals = []
+    session = kwargs.pop("session", None)
+    run_kwargs = kwargs.pop("run_kwargs", dict())
     if evals:
         for eval_dmatrix, name in evals:
             if not isinstance(name, str):
@@ -110,12 +121,11 @@ def train(params, dtrain, evals=None, evals_result=None, num_class=None, **kwarg
                 processed_evals.append((eval_dmatrix, name))
             else:
                 processed_evals.append((to_dmatrix(eval_dmatrix), name))
-
     return XGBTrain(
         params=params,
         dtrain=dtrain,
         evals=processed_evals,
         evals_result=evals_result,
         num_class=num_class,
-        **kwargs
-    )()
+        **kwargs,
+    )(evals_result).execute(session=session, **run_kwargs)

maxframe/{core/operator/fuse.py → learn/core.py}

@@ -12,18 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ... import opcodes
-from ...serialization.serializables import ReferenceField
-from ..graph import ChunkGraph
-from .base import Operator
+from ..core.entity.objects import Object, ObjectData
 
 
-class Fuse(Operator):
-    __slots__ = ("_fuse_graph",)
-    _op_type_ = opcodes.FUSE
+class ModelData(ObjectData):
+    pass
 
-    fuse_graph = ReferenceField("fuse_graph", ChunkGraph)
 
+class Model(Object):
+    pass
 
-class FuseChunkMixin:
-    __slots__ = ()
+
+MODEL_TYPE = (Model, ModelData)

maxframe/protocol.py

@@ -15,7 +15,7 @@
 import base64
 import enum
 import uuid
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
+from typing import Any, Dict, Generic, List, Optional, Type, TypeVar
 
 import pandas as pd
 
@@ -38,7 +38,6 @@ from .serialization.serializables import (
     Serializable,
     SeriesField,
     StringField,
-    TupleField,
 )
 
 pickling_support.install()
@@ -92,19 +91,6 @@ class DataSerializeType(enum.Enum):
     PICKLE = 0
 
 
-class VolumeDataMeta(Serializable):
-    output_type: OutputType = EnumField(
-        "output_type", OutputType, FieldTypes.int8, default=None
-    )
-    serial_type: DataSerializeType = EnumField(
-        "serial_type", DataSerializeType, FieldTypes.int8, default=None
-    )
-    shape: Tuple[int, ...] = TupleField("shape", FieldTypes.int64, default=None)
-    nsplits: Tuple[Tuple[int, ...], ...] = TupleField(
-        "nsplits", FieldTypes.tuple(FieldTypes.tuple(FieldTypes.int64)), default=None
-    )
-
-
 _result_type_to_info_cls: Dict[ResultType, Type["ResultInfo"]] = dict()
 
 
@@ -154,6 +140,9 @@ class ODPSTableResultInfo(ResultInfo):
     partition_specs: Optional[List[str]] = ListField(
         "partition_specs", FieldTypes.string, default=None
     )
+    table_meta: Optional["DataFrameTableMeta"] = ReferenceField(
+        "table_meta", default=None
+    )
 
     def __init__(self, result_type: ResultType = None, **kw):
         result_type = result_type or ResultType.ODPS_TABLE
@@ -164,8 +153,17 @@ class ODPSTableResultInfo(ResultInfo):
         ret["full_table_name"] = self.full_table_name
         if self.partition_specs:
             ret["partition_specs"] = self.partition_specs
+        if self.table_meta:
+            ret["table_meta"] = self.table_meta.to_json()
         return ret
 
+    @classmethod
+    def _json_to_kwargs(cls, serialized: dict) -> dict:
+        kw = super()._json_to_kwargs(serialized)
+        if "table_meta" in kw:
+            kw["table_meta"] = DataFrameTableMeta.from_json(kw["table_meta"])
+        return kw
+
 
 class ODPSVolumeResultInfo(ResultInfo):
     _result_type = ResultType.ODPS_VOLUME
@@ -469,7 +467,7 @@ class DecrefRequest(Serializable):
     keys: List[str] = ListField("keys", FieldTypes.string, default=None)
 
 
-class DataFrameTableMeta(Serializable):
+class DataFrameTableMeta(JsonSerializable):
     __slots__ = "_pd_column_names", "_pd_index_level_names"
 
     table_name: Optional[str] = StringField("table_name", default=None)
@@ -500,7 +498,7 @@ class DataFrameTableMeta(Serializable):
             self._pd_index_level_names = self.pd_index_dtypes.index.tolist()
             return self._pd_index_level_names
 
-    def __eq__(self, other: "Serializable") -> bool:
+    def __eq__(self, other: "DataFrameTableMeta") -> bool:
         if not isinstance(other, type(self)):
             return False
         for k in self._FIELDS:
@@ -511,3 +509,29 @@ class DataFrameTableMeta(Serializable):
             if not is_same:
                 return False
         return True
+
+    def to_json(self) -> dict:
+        b64_pk = lambda x: base64.b64encode(pickle.dumps(x))
+        ret = {
+            "table_name": self.table_name,
+            "type": self.type.value,
+            "table_column_names": self.table_column_names,
+            "table_index_column_names": self.table_index_column_names,
+            "pd_column_dtypes": b64_pk(self.pd_column_dtypes),
+            "pd_column_level_names": b64_pk(self.pd_column_level_names),
+            "pd_index_dtypes": b64_pk(self.pd_index_dtypes),
+        }
+        return ret
+
+    @classmethod
+    def from_json(cls, serialized: dict) -> "DataFrameTableMeta":
+        b64_upk = lambda x: pickle.loads(base64.b64decode(x))
+        serialized.update(
+            {
+                "type": OutputType(serialized["type"]),
+                "pd_column_dtypes": b64_upk(serialized["pd_column_dtypes"]),
+                "pd_column_level_names": b64_upk(serialized["pd_column_level_names"]),
+                "pd_index_dtypes": b64_upk(serialized["pd_index_dtypes"]),
+            }
+        )
+        return DataFrameTableMeta(**serialized)

maxframe/remote/core.py

@@ -15,7 +15,7 @@
 from functools import partial
 
 from .. import opcodes
-from ..core import ENTITY_TYPE, ChunkData
+from ..core import ENTITY_TYPE
 from ..core.operator import ObjectOperator, ObjectOperatorMixin
 from ..dataframe.core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
 from ..serialization.serializables import (
@@ -26,7 +26,7 @@ from ..serialization.serializables import (
     ListField,
 )
 from ..tensor.core import TENSOR_TYPE
-from ..utils import build_fetch_tileable, find_objects, replace_objects
+from ..utils import find_objects, replace_objects
 
 
 class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
@@ -63,12 +63,8 @@ class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
         if raw_inputs is not None:
             for raw_inp in raw_inputs:
                 if self._no_prepare(raw_inp):
-                    if not isinstance(self._inputs[0], ChunkData):
-                        # not in tile, set_inputs from tileable
-                        mapping[raw_inp] = next(function_inputs)
-                    else:
-                        # in tile, set_inputs from chunk
-                        mapping[raw_inp] = build_fetch_tileable(raw_inp)
+                    # not in tile, set_inputs from tileable
+                    mapping[raw_inp] = next(function_inputs)
                 else:
                     mapping[raw_inp] = next(function_inputs)
         self.function_args = replace_objects(self.function_args, mapping)

maxframe/serialization/__init__.py

@@ -17,6 +17,7 @@ from .core import (
     PickleContainer,
     Serializer,
     deserialize,
+    load_type,
     pickle_buffers,
     serialize,
     serialize_with_spawn,

maxframe/serialization/serializables/core.py

@@ -51,7 +51,10 @@ def _is_field_primitive_compound(field: Field):
 class SerializableMeta(type):
     def __new__(mcs, name: str, bases: Tuple[Type], properties: Dict):
         # All the fields including misc fields.
-        name_hash = hash(f"{properties.get('__module__')}.{name}")
+        legacy_name_hash = hash(f"{properties.get('__module__')}.{name}")
+        name_hash = hash(
+            f"{properties.get('__module__')}.{properties.get('__qualname__')}"
+        )
         all_fields = dict()
         # mapping field names to base classes
         field_to_cls_hash = dict()
@@ -107,6 +110,10 @@ class SerializableMeta(type):
         slots.update(properties_field_slot_names)
 
         properties = properties_without_fields
+
+        # todo remove this prop when all versions below v1.0.0rc1 is eliminated
+        properties["_LEGACY_NAME_HASH"] = legacy_name_hash
+
         properties["_NAME_HASH"] = name_hash
         properties["_FIELDS"] = all_fields
         properties["_FIELD_ORDER"] = field_order
@@ -210,8 +217,8 @@ class SerializableSerializer(Serializer):
     """
 
     @classmethod
-    def _get_obj_field_count_key(cls, obj: Serializable):
-        return f"FC_{obj._NAME_HASH}"
+    def _get_obj_field_count_key(cls, obj: Serializable, legacy: bool = False):
+        return f"FC_{obj._NAME_HASH if not legacy else obj._LEGACY_NAME_HASH}"
 
     @classmethod
     def _get_field_values(cls, obj: Serializable, fields):
@@ -290,6 +297,12 @@ class SerializableSerializer(Serializer):
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
             server_fields = obj_class._NON_PRIMITIVE_FIELDS
 
+        legacy_to_new_hash = {
+            c._LEGACY_NAME_HASH: c._NAME_HASH
+            for c in obj_class.__mro__
+            if hasattr(c, "_NAME_HASH") and c._LEGACY_NAME_HASH != c._NAME_HASH
+        }
+
         if client_cls_to_field_count:
             field_num, server_field_num = 0, 0
             for cls_hash, count in client_cls_to_field_count.items():
@@ -301,20 +314,40 @@ class SerializableSerializer(Serializer):
                     if not is_primitive or value != {}:
                         cls._set_field_value(obj, field, value)
                 field_num += count
-                server_field_num += server_cls_to_field_count[cls_hash]
+                try:
+                    server_field_num += server_cls_to_field_count[cls_hash]
+                except KeyError:
+                    try:
+                        # todo remove this fallback when all
+                        #  versions below v1.0.0rc1 is eliminated
+                        server_field_num += server_cls_to_field_count[
+                            legacy_to_new_hash[cls_hash]
+                        ]
+                    except KeyError:
+                        # it is possible that certain type of field does not exist
+                        #  at server side
+                        pass
         else:
+            # handle legacy serialization style, with all fields sorted by name
             # todo remove this branch when all versions below v0.1.0b5 is eliminated
             from .field import AnyField
 
-            # legacy serialization style, with all fields sorted by name
             if is_primitive:
-                field_attr = "_legacy_deprecated_primitives"
+                new_field_attr = "_legacy_new_primitives"
+                deprecated_field_attr = "_legacy_deprecated_primitives"
             else:
-                field_attr = "_legacy_deprecated_non_primitives"
+                new_field_attr = "_legacy_new_non_primitives"
+                deprecated_field_attr = "_legacy_deprecated_non_primitives"
+
+            # remove fields added on later releases
+            new_names = set(getattr(obj_class, new_field_attr, None) or [])
+            server_fields = [f for f in server_fields if f.name not in new_names]
+
+            # fill fields deprecated on later releases
             deprecated_fields = []
             deprecated_names = set()
-            if hasattr(obj_class, field_attr):
-                deprecated_names = set(getattr(obj_class, field_attr))
+            if hasattr(obj_class, deprecated_field_attr):
+                deprecated_names = set(getattr(obj_class, deprecated_field_attr))
                 for field_name in deprecated_names:
                     field = AnyField(tag=field_name)
                     field.name = field_name
@@ -342,6 +375,12 @@ class SerializableSerializer(Serializer):
         field_count_data = self.get_public_data(
             context, self._get_obj_field_count_key(obj)
         )
+        if field_count_data is None:
+            # todo remove this fallback when all
+            #  versions below v1.0.0rc1 is eliminated
+            field_count_data = self.get_public_data(
+                context, self._get_obj_field_count_key(obj, legacy=True)
+            )
         if field_count_data is not None:
             cls_to_prim_key, cls_to_non_prim_key = msgpack.loads(field_count_data)
             cls_to_prim_key = dict(cls_to_prim_key)

maxframe/tensor/__init__.py

@@ -114,7 +114,6 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,
@@ -143,7 +142,16 @@ from .indexing import (
     take,
     unravel_index,
 )
-from .merge import stack
+from .merge import concatenate, stack, vstack
+from .misc import (
+    atleast_1d,
+    atleast_2d,
+    atleast_3d,
+    broadcast_to,
+    transpose,
+    unique,
+    where,
+)
 from .rechunk import rechunk
 from .reduction import (
     all,
@@ -180,4 +188,63 @@ from .reduction import std, sum, var
 from .reshape import reshape
 from .ufunc import ufunc
 
+# isort: off
+# noinspection PyUnresolvedReferences
+from numpy import (
+    NAN,
+    NINF,
+    AxisError,
+    Inf,
+    NaN,
+    e,
+    errstate,
+    geterr,
+    inf,
+    nan,
+    newaxis,
+    pi,
+    seterr,
+)
+
+# import numpy types
+# noinspection PyUnresolvedReferences
+from numpy import (
+    bool_ as bool,
+    bytes_,
+    cfloat,
+    character,
+    complex64,
+    complex128,
+    complexfloating,
+    datetime64,
+    double,
+    dtype,
+    flexible,
+    float16,
+    float32,
+    float64,
+    floating,
+    generic,
+    inexact,
+    int8,
+    int16,
+    int32,
+    int64,
+    intc,
+    intp,
+    number,
+    integer,
+    object_ as object,
+    signedinteger,
+    timedelta64,
+    uint,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    unicode_,
+    unsignedinteger,
+    void,
+)
+
 del fetch, ufunc

maxframe/tensor/arithmetic/isclose.py

@@ -23,6 +23,7 @@ from .core import TensorBinOp
 
 class TensorIsclose(TensorBinOp):
     _op_type_ = opcodes.ISCLOSE
+    _func_name = "isclose"
 
     rtol = Float64Field("rtol", default=None)
     atol = Float64Field("atol", default=None)

maxframe/tensor/arithmetic/tests/test_arithmetic.py

@@ -17,26 +17,13 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.arithmetic.core import TensorBinOp, TensorUnaryOp
+from maxframe.utils import collect_leaf_operators
+
 from ....core import enter_mode
 from ...core import SparseTensor, Tensor
 from ...datasource import array, empty, ones, tensor
-from .. import (
-    TensorAdd,
-    TensorGreaterThan,
-    TensorIsclose,
-    TensorLog,
-    TensorSubtract,
-    add,
-    around,
-    cos,
-    frexp,
-    isclose,
-    isfinite,
-    log,
-    negative,
-    subtract,
-    truediv,
-)
+from .. import *  # noqa: F401
 
 
 def test_add():
@@ -412,3 +399,20 @@ def test_build_mode():
 
     with enter_mode(build=True):
         assert t1 != 2
+
+
+def test_unary_op_func_name():
+    # make sure all the unary op has defined the func name.
+
+    results = collect_leaf_operators(TensorUnaryOp)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")
+
+
+def test_binary_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorBinOp)
+    for op_type in results:
+        if op_type not in (TensorSetImag, TensorSetReal):
+            assert hasattr(op_type, "_func_name")