maxframe 0.1.0b4__cp310-cp310-win_amd64.whl → 1.0.0rc1__cp310-cp310-win_amd64.whl

maxframe/serialization/core.pyx

@@ -130,11 +130,30 @@ cdef Serializer get_deserializer(int32_t deserializer_id):
 
 cdef class Serializer:
     serializer_id = None
+    _public_data_context_key = 0x7fffffff - 1
 
     def __cinit__(self):
         # make the value can be referenced with C code
         self._serializer_id = self.serializer_id
 
+    cpdef bint is_public_data_exist(self, dict context, object key):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            return False
+        return key in public_dict
+
+    cpdef put_public_data(self, dict context, object key, object value):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            public_dict = context[self._public_data_context_key] = {}
+        public_dict[key] = value
+
+    cpdef get_public_data(self, dict context, object key):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            return None
+        return public_dict.get(key)
+
     cpdef serial(self, object obj, dict context):
         """
         Returns intermediate serialization result of certain object.
@@ -993,17 +1012,20 @@ def serialize(obj, dict context = None):
     cdef list subs
     cdef bint final
     cdef _IdContextHolder id_context_holder = _IdContextHolder()
+    cdef tuple result
 
     context = context if context is not None else dict()
     serialized, subs, final = _serial_single(obj, context, id_context_holder)
     if final or not subs:
         # marked as a leaf node, return directly
-        return [{}, serialized], subs
-
-    serial_stack.append(_SerialStackItem(serialized, subs))
-    return _serialize_with_stack(
-        serial_stack, None, context, id_context_holder, result_bufs_list
-    )
+        result = [{}, serialized], subs
+    else:
+        serial_stack.append(_SerialStackItem(serialized, subs))
+        result = _serialize_with_stack(
+            serial_stack, None, context, id_context_holder, result_bufs_list
+        )
+    result[0][0]["_PUB"] = context.get(Serializer._public_data_context_key)
+    return result
 
 
 async def serialize_with_spawn(
@@ -1036,31 +1058,38 @@ async def serialize_with_spawn(
     cdef list subs
     cdef bint final
     cdef _IdContextHolder id_context_holder = _IdContextHolder()
+    cdef tuple result
 
     context = context if context is not None else dict()
     serialized, subs, final = _serial_single(obj, context, id_context_holder)
     if final or not subs:
         # marked as a leaf node, return directly
-        return [{}, serialized], subs
-
-    serial_stack.append(_SerialStackItem(serialized, subs))
+        result = [{}, serialized], subs
+    else:
+        serial_stack.append(_SerialStackItem(serialized, subs))
 
-    try:
-        result = _serialize_with_stack(
-            serial_stack, None, context, id_context_holder, result_bufs_list, spawn_threshold
-        )
-    except _SerializeObjectOverflow as ex:
-        result = await asyncio.get_running_loop().run_in_executor(
-            executor,
-            _serialize_with_stack,
-            serial_stack,
-            ex.cur_serialized,
-            context,
-            id_context_holder,
-            result_bufs_list,
-            0,
-            ex.num_total_serialized,
-        )
+        try:
+            result = _serialize_with_stack(
+                serial_stack,
+                None,
+                context,
+                id_context_holder,
+                result_bufs_list,
+                spawn_threshold,
+            )
+        except _SerializeObjectOverflow as ex:
+            result = await asyncio.get_running_loop().run_in_executor(
+                executor,
+                _serialize_with_stack,
+                serial_stack,
+                ex.cur_serialized,
+                context,
+                id_context_holder,
+                result_bufs_list,
+                0,
+                ex.num_total_serialized,
+            )
+    result[0][0]["_PUB"] = context.get(Serializer._public_data_context_key)
     return result
 
 

maxframe/serialization/exception.py

@@ -35,7 +35,7 @@ class RemoteException(MaxFrameError):
     def from_exception(cls, exc: Exception):
         try:
             buffers = pickle_buffers(exc)
-        except (TypeError, pickle.PicklingError):
+        except:
             logger.exception("Cannot pickle exception %s", exc)
             buffers = []
 

maxframe/serialization/pandas.py

@@ -176,11 +176,16 @@ class PdTimestampSerializer(Serializer):
 
 class PdTimedeltaSerializer(Serializer):
     def serial(self, obj: pd.Timedelta, context: Dict):
-        return [int(obj.seconds), obj.microseconds, obj.nanoseconds], [], True
+        return [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days], [], True
 
     def deserial(self, serialized: List, context: Dict, subs: List):
+        days = 0 if len(serialized) < 4 else serialized[3]
+        seconds, microseconds, nanoseconds = serialized[:3]
         return pd.Timedelta(
-            seconds=serialized[0], microseconds=serialized[1], nanoseconds=serialized[2]
+            days=days,
+            seconds=seconds,
+            microseconds=microseconds,
+            nanoseconds=nanoseconds,
         )
 
 

maxframe/serialization/serializables/core.py

@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import operator
 import weakref
-from typing import Dict, List, Tuple, Type
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
 
+from ...lib.mmh3 import hash
 from ..core import Placeholder, Serializer, buffered, load_type
 from .field import Field
 from .field_type import DictType, ListType, PrimitiveFieldType, TupleType
@@ -50,11 +51,16 @@ def _is_field_primitive_compound(field: Field):
 class SerializableMeta(type):
     def __new__(mcs, name: str, bases: Tuple[Type], properties: Dict):
         # All the fields including misc fields.
+        name_hash = hash(f"{properties.get('__module__')}.{name}")
         all_fields = dict()
+        # mapping field names to base classes
+        field_to_cls_hash = dict()
 
         for base in bases:
-            if hasattr(base, "_FIELDS"):
-                all_fields.update(base._FIELDS)
+            if not hasattr(base, "_FIELDS"):
+                continue
+            all_fields.update(base._FIELDS)
+            field_to_cls_hash.update(base._FIELD_TO_NAME_HASH)
 
         properties_without_fields = {}
         properties_field_slot_names = []
@@ -64,6 +70,8 @@ class SerializableMeta(type):
                 continue
 
             field = all_fields.get(k)
+            # record the field for the class being created
+            field_to_cls_hash[k] = name_hash
             if field is None:
                 properties_field_slot_names.append(k)
             else:
@@ -75,23 +83,40 @@ class SerializableMeta(type):
 
         # Make field order deterministic to serialize it as list instead of dict.
         field_order = list(all_fields)
-        all_fields = dict(sorted(all_fields.items(), key=operator.itemgetter(0)))
         primitive_fields = []
+        primitive_field_names = set()
         non_primitive_fields = []
-        for v in all_fields.values():
+        for field_name, v in all_fields.items():
             if _is_field_primitive_compound(v):
                 primitive_fields.append(v)
+                primitive_field_names.add(field_name)
             else:
                 non_primitive_fields.append(v)
 
+        # count number of fields for every base class
+        cls_to_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        for field_name in field_order:
+            cls_hash = field_to_cls_hash[field_name]
+            if field_name in primitive_field_names:
+                cls_to_primitive_field_count[cls_hash] += 1
+            else:
+                cls_to_non_primitive_field_count[cls_hash] += 1
+
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
 
         properties = properties_without_fields
+        properties["_NAME_HASH"] = name_hash
         properties["_FIELDS"] = all_fields
         properties["_FIELD_ORDER"] = field_order
+        properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+            cls_to_non_primitive_field_count
+        )
         properties["__slots__"] = tuple(slots)
 
         clz = type.__new__(mcs, name, bases, properties)
@@ -114,10 +139,14 @@ class Serializable(metaclass=SerializableMeta):
     _cache_primitive_serial = False
     _ignore_non_existing_keys = False
 
+    _NAME_HASH: int
     _FIELDS: Dict[str, Field]
     _FIELD_ORDER: List[str]
+    _FIELD_TO_NAME_HASH: Dict[str, int]
     _PRIMITIVE_FIELDS: List[str]
+    _CLS_TO_PRIMITIVE_FIELD_COUNT: Dict[int, int]
     _NON_PRIMITIVE_FIELDS: List[str]
+    _CLS_TO_NON_PRIMITIVE_FIELD_COUNT: Dict[int, int]
 
     def __init__(self, *args, **kwargs):
         fields = self._FIELDS
@@ -180,6 +209,10 @@ class SerializableSerializer(Serializer):
     Leverage DictSerializer to perform serde.
     """
 
+    @classmethod
+    def _get_obj_field_count_key(cls, obj: Serializable):
+        return f"FC_{obj._NAME_HASH}"
+
     @classmethod
     def _get_field_values(cls, obj: Serializable, fields):
         values = []
@@ -210,6 +243,18 @@ class SerializableSerializer(Serializer):
 
         compound_vals = self._get_field_values(obj, obj._NON_PRIMITIVE_FIELDS)
         cls_module = f"{type(obj).__module__}#{type(obj).__qualname__}"
+
+        field_count_key = self._get_obj_field_count_key(obj)
+        if not self.is_public_data_exist(context, field_count_key):
+            # store field distribution for current Serializable
+            counts = [
+                list(obj._CLS_TO_PRIMITIVE_FIELD_COUNT.items()),
+                list(obj._CLS_TO_NON_PRIMITIVE_FIELD_COUNT.items()),
+            ]
+            field_count_data = msgpack.dumps(counts)
+            self.put_public_data(
+                context, self._get_obj_field_count_key(obj), field_count_data
+            )
         return [cls_module, primitive_vals], [compound_vals], False
 
     @staticmethod
@@ -229,6 +274,62 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _set_field_values(
+        cls,
+        obj: Serializable,
+        values: List[Any],
+        client_cls_to_field_count: Optional[Dict[str, int]],
+        is_primitive: bool = True,
+    ):
+        obj_class = type(obj)
+        if is_primitive:
+            server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
+            server_fields = obj_class._PRIMITIVE_FIELDS
+        else:
+            server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
+            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+
+        if client_cls_to_field_count:
+            field_num, server_field_num = 0, 0
+            for cls_hash, count in client_cls_to_field_count.items():
+                # cut values and fields given field distribution
+                # at client and server end
+                cls_fields = server_fields[server_field_num : field_num + count]
+                cls_values = values[field_num : field_num + count]
+                for field, value in zip(cls_fields, cls_values):
+                    if not is_primitive or value != {}:
+                        cls._set_field_value(obj, field, value)
+                field_num += count
+                server_field_num += server_cls_to_field_count[cls_hash]
+        else:
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            from .field import AnyField
+
+            # legacy serialization style, with all fields sorted by name
+            if is_primitive:
+                field_attr = "_legacy_deprecated_primitives"
+            else:
+                field_attr = "_legacy_deprecated_non_primitives"
+            deprecated_fields = []
+            deprecated_names = set()
+            if hasattr(obj_class, field_attr):
+                deprecated_names = set(getattr(obj_class, field_attr))
+                for field_name in deprecated_names:
+                    field = AnyField(tag=field_name)
+                    field.name = field_name
+                    deprecated_fields.append(field)
+            server_fields = sorted(
+                server_fields + deprecated_fields, key=lambda f: f.name
+            )
+            for field, value in zip(server_fields, values):
+                if not is_primitive or value != {}:
+                    try:
+                        cls._set_field_value(obj, field, value)
+                    except AttributeError:  # pragma: no cover
+                        if field.name not in deprecated_names:
+                            raise
+
     def deserial(self, serialized: List, context: Dict, subs: List) -> Serializable:
         obj_class_name, primitives = serialized
         obj_class = load_type(obj_class_name, Serializable)
@@ -238,14 +339,20 @@ class SerializableSerializer(Serializer):
 
         obj = obj_class.__new__(obj_class)
 
-        if primitives:
-            for field, value in zip(obj_class._PRIMITIVE_FIELDS, primitives):
-                if value != {}:
-                    self._set_field_value(obj, field, value)
+        field_count_data = self.get_public_data(
+            context, self._get_obj_field_count_key(obj)
+        )
+        if field_count_data is not None:
+            cls_to_prim_key, cls_to_non_prim_key = msgpack.loads(field_count_data)
+            cls_to_prim_key = dict(cls_to_prim_key)
+            cls_to_non_prim_key = dict(cls_to_non_prim_key)
+        else:
+            cls_to_prim_key, cls_to_non_prim_key = None, None
 
+        if primitives:
+            self._set_field_values(obj, primitives, cls_to_prim_key, True)
         if obj_class._NON_PRIMITIVE_FIELDS:
-            for field, value in zip(obj_class._NON_PRIMITIVE_FIELDS, subs[0]):
-                self._set_field_value(obj, field, value)
+            self._set_field_values(obj, subs[0], cls_to_non_prim_key, False)
         obj.__on_deserialize__()
         return obj
 

maxframe/serialization/serializables/tests/test_serializable.py

@@ -94,6 +94,11 @@ class MySimpleSerializable(Serializable):
     _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
 
 
+class MySubSerializable(MySimpleSerializable):
+    _m_int_val = Int64Field("m_int_val", default=250)
+    _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+
 class MySerializable(Serializable):
     _id = IdentityField("id")
     _any_val = AnyField("any_val")
@@ -141,7 +146,7 @@ class MySerializable(Serializable):
 
 
 @pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
-@switch_unpickle
+@switch_unpickle(forbidden=False)
 def test_serializable(set_is_ci):
     my_serializable = MySerializable(
         _id="1",
@@ -165,7 +170,9 @@ def test_serializable(set_is_ci):
         _key_val=MyHasKey("aaa"),
         _ndarray_val=np.random.rand(4, 3),
         _datetime64_val=pd.Timestamp(123),
-        _timedelta64_val=pd.Timedelta(days=1),
+        _timedelta64_val=pd.Timedelta(
+            days=1, seconds=123, microseconds=345, nanoseconds=132
+        ),
         _datatype_val=np.dtype(np.int32),
         _index_val=pd.Index([1, 2]),
         _series_val=pd.Series(["a", "b"]),
@@ -187,9 +194,44 @@ def test_serializable(set_is_ci):
     _assert_serializable_eq(my_serializable, my_serializable2)
 
 
+@pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
+@switch_unpickle
+def test_compatible_serializable(set_is_ci):
+    global MySimpleSerializable, MySubSerializable
+
+    old_base, old_sub = MySimpleSerializable, MySubSerializable
+
+    try:
+        my_sub_serializable = MySubSerializable(
+            _id="id_val",
+            _list_val=["abcd", "wxyz"],
+            _ref_val=MyHasKey(),
+            _m_int_val=3412,
+            _m_str_val="dfgghj",
+        )
+        header, buffers = serialize(my_sub_serializable)
+
+        class MySimpleSerializable(Serializable):
+            _id = IdentityField("id")
+            _int_val = Int64Field("int_val", default=1000)
+            _list_val = ListField("list_val", default_factory=list)
+            _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
+            _dict_val = DictField("dict_val")
+
+        class MySubSerializable(MySimpleSerializable):
+            _m_int_val = Int64Field("m_int_val", default=250)
+            _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+        my_sub_serializable2 = deserialize(header, buffers)
+        assert type(my_sub_serializable) is not type(my_sub_serializable2)
+        _assert_serializable_eq(my_sub_serializable, my_sub_serializable2)
+    finally:
+        MySimpleSerializable, MySubSerializable = old_base, old_sub
+
+
 def _assert_serializable_eq(my_serializable, my_serializable2):
     for field_name, field in my_serializable._FIELDS.items():
-        if not hasattr(my_serializable, field.tag):
+        if not hasattr(my_serializable, field.name):
             continue
         expect_value = getattr(my_serializable, field_name)
         actual_value = getattr(my_serializable2, field_name)
@@ -208,7 +250,7 @@ def _assert_serializable_eq(my_serializable, my_serializable2):
         elif callable(expect_value):
             assert expect_value(1) == actual_value(1)
         else:
-            assert expect_value == actual_value
+            assert expect_value == actual_value, f"Field {field_name}"
 
 
 @pytest.mark.parametrize("set_is_ci", [True], indirect=True)

maxframe/session.py

@@ -365,6 +365,15 @@ class AbstractAsyncSession(AbstractSession, metaclass=ABCMeta):
         Stop server.
         """
 
+    @abstractmethod
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get Logview address
+        Returns
+        -------
+            Logview address
+        """
+
     def close(self):
         asyncio.run(self.destroy())
 
@@ -549,6 +558,15 @@ class AbstractSyncSession(AbstractSession, metaclass=ABCMeta):
 
         return fetch(tileables, self, offsets=offsets, sizes=sizes)
 
+    @abstractmethod
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get logview address
+        Returns
+        -------
+            logview address
+        """
+
 
 def _delegate_to_isolated_session(func: Union[Callable, Coroutine]):
     if asyncio.iscoroutinefunction(func):
@@ -728,6 +746,11 @@ class AsyncSession(AbstractAsyncSession):
         await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(coro, self._loop))
         stop_isolation()
 
+    @implements(AbstractAsyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
 
 class ProgressBar:
     def __init__(self, show_progress):
@@ -949,6 +972,11 @@ class SyncSession(AbstractSyncSession):
     def get_cluster_versions(self) -> List[str]:
         pass  # pragma: no cover
 
+    @implements(AbstractSyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
     def destroy(self):
         coro = self._isolated_session.destroy()
         asyncio.run_coroutine_threadsafe(coro, self._loop).result()

maxframe/tensor/__init__.py

@@ -114,7 +114,7 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, where
+from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,

maxframe/tensor/arithmetic/tests/test_arithmetic.py

@@ -252,7 +252,7 @@ def test_compare():
 
 def test_frexp():
     t1 = ones((3, 4, 5), chunk_size=2)
-    t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2)
+    t2 = empty((3, 4, 5), dtype=np.dtype(float), chunk_size=2)
     op_type = type(t1.op)
 
     o1, o2 = frexp(t1)

maxframe/tensor/base/__init__.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 from .astype import TensorAstype
+from .atleast_1d import atleast_1d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose
+from .unique import unique
 from .where import TensorWhere, where
 
 

maxframe/tensor/base/atleast_1d.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_1d(*tensors):
+    """
+    Convert inputs to tensors with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional tensors, whilst
+    higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more input tensors.
+
+    Returns
+    -------
+    ret : Tensor
+        An tensor, or list of tensors, each with ``a.ndim >= 1``.
+        Copies are made only if necessary.
+
+    See Also
+    --------
+    atleast_2d, atleast_3d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_1d(1.0).execute()
+    array([ 1.])
+
+    >>> x = mt.arange(9.0).reshape(3,3)
+    >>> mt.atleast_1d(x).execute()
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4.,  5.],
+           [ 6.,  7.,  8.]])
+    >>> mt.atleast_1d(x) is x
+    True
+
+    >>> mt.atleast_1d(1, [3, 4]).execute()
+    [array([1]), array([3, 4])]
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)