maxframe 0.1.0b4__cp310-cp310-win32.whl → 1.0.0__cp310-cp310-win32.whl

maxframe/serialization/pandas.py

@@ -134,8 +134,10 @@ class ArraySerializer(Serializer):
                 data_parts = [obj.tolist()]
             else:
                 data_parts = [obj.to_numpy().tolist()]
-        else:
+        elif hasattr(obj, "_data"):
             data_parts = [getattr(obj, "_data")]
+        else:
+            data_parts = [getattr(obj, "_pa_array")]
         return [ser_type], [dtype] + data_parts, False
 
     def deserial(self, serialized: List, context: Dict, subs: List):
@@ -155,33 +157,66 @@ class PdTimestampSerializer(Serializer):
         else:
             zone_info = []
             ts = obj.to_pydatetime().timestamp()
-        return (
-            [int(ts), obj.microsecond, obj.nanosecond],
-            zone_info,
-            bool(zone_info),
-        )
+        elements = [int(ts), obj.microsecond, obj.nanosecond]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, zone_info, bool(zone_info)
 
     def deserial(self, serialized: List, context: Dict, subs: List):
         if subs:
-            val = pd.Timestamp.utcfromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
-            val = val.replace(tzinfo=datetime.timezone.utc).tz_convert(subs[0])
+            pydt = datetime.datetime.utcfromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+                "tzinfo": datetime.timezone.utc,
+            }
+            if len(serialized) > 3:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs).tz_convert(subs[0])
         else:
-            val = pd.Timestamp.fromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
+            pydt = datetime.datetime.fromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+            }
+            if len(serialized) >= 4:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs)
         return val
 
 
 class PdTimedeltaSerializer(Serializer):
     def serial(self, obj: pd.Timedelta, context: Dict):
-        return [int(obj.seconds), obj.microseconds, obj.nanoseconds], [], True
+        elements = [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, [], True
 
     def deserial(self, serialized: List, context: Dict, subs: List):
-        return pd.Timedelta(
-            seconds=serialized[0], microseconds=serialized[1], nanoseconds=serialized[2]
-        )
+        days = 0 if len(serialized) < 4 else serialized[3]
+        unit = None if len(serialized) < 5 else serialized[4]
+        seconds, microseconds, nanoseconds = serialized[:3]
+        kwargs = {
+            "days": days,
+            "seconds": seconds,
+            "microseconds": microseconds,
+            "nanoseconds": nanoseconds,
+        }
+        if unit is not None:
+            kwargs["unit"] = unit
+        return pd.Timedelta(**kwargs)
 
 
 class NoDefaultSerializer(Serializer):

maxframe/serialization/serializables/core.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import operator
 import weakref
-from typing import Dict, List, Tuple, Type
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
 
+from ...lib.mmh3 import hash
+from ...utils import no_default
 from ..core import Placeholder, Serializer, buffered, load_type
 from .field import Field
 from .field_type import DictType, ListType, PrimitiveFieldType, TupleType
@@ -50,11 +52,19 @@ def _is_field_primitive_compound(field: Field):
 class SerializableMeta(type):
     def __new__(mcs, name: str, bases: Tuple[Type], properties: Dict):
         # All the fields including misc fields.
+        legacy_name_hash = hash(f"{properties.get('__module__')}.{name}")
+        name_hash = hash(
+            f"{properties.get('__module__')}.{properties.get('__qualname__')}"
+        )
         all_fields = dict()
+        # mapping field names to base classes
+        field_to_cls_hash = dict()
 
         for base in bases:
-            if hasattr(base, "_FIELDS"):
-                all_fields.update(base._FIELDS)
+            if not hasattr(base, "_FIELDS"):
+                continue
+            all_fields.update(base._FIELDS)
+            field_to_cls_hash.update(base._FIELD_TO_NAME_HASH)
 
         properties_without_fields = {}
         properties_field_slot_names = []
@@ -64,6 +74,8 @@ class SerializableMeta(type):
                 continue
 
             field = all_fields.get(k)
+            # record the field for the class being created
+            field_to_cls_hash[k] = name_hash
             if field is None:
                 properties_field_slot_names.append(k)
             else:
@@ -75,23 +87,44 @@ class SerializableMeta(type):
 
         # Make field order deterministic to serialize it as list instead of dict.
         field_order = list(all_fields)
-        all_fields = dict(sorted(all_fields.items(), key=operator.itemgetter(0)))
         primitive_fields = []
+        primitive_field_names = set()
         non_primitive_fields = []
-        for v in all_fields.values():
+        for field_name, v in all_fields.items():
             if _is_field_primitive_compound(v):
                 primitive_fields.append(v)
+                primitive_field_names.add(field_name)
             else:
                 non_primitive_fields.append(v)
 
+        # count number of fields for every base class
+        cls_to_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        for field_name in field_order:
+            cls_hash = field_to_cls_hash[field_name]
+            if field_name in primitive_field_names:
+                cls_to_primitive_field_count[cls_hash] += 1
+            else:
+                cls_to_non_primitive_field_count[cls_hash] += 1
+
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
 
         properties = properties_without_fields
+
+        # todo remove this prop when all versions below v1.0.0rc1 is eliminated
+        properties["_LEGACY_NAME_HASH"] = legacy_name_hash
+
+        properties["_NAME_HASH"] = name_hash
         properties["_FIELDS"] = all_fields
         properties["_FIELD_ORDER"] = field_order
+        properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+            cls_to_non_primitive_field_count
+        )
         properties["__slots__"] = tuple(slots)
 
         clz = type.__new__(mcs, name, bases, properties)
@@ -114,10 +147,14 @@ class Serializable(metaclass=SerializableMeta):
     _cache_primitive_serial = False
     _ignore_non_existing_keys = False
 
+    _NAME_HASH: int
     _FIELDS: Dict[str, Field]
     _FIELD_ORDER: List[str]
+    _FIELD_TO_NAME_HASH: Dict[str, int]
     _PRIMITIVE_FIELDS: List[str]
+    _CLS_TO_PRIMITIVE_FIELD_COUNT: Dict[int, int]
     _NON_PRIMITIVE_FIELDS: List[str]
+    _CLS_TO_NON_PRIMITIVE_FIELD_COUNT: Dict[int, int]
 
     def __init__(self, *args, **kwargs):
         fields = self._FIELDS
@@ -175,11 +212,31 @@ class _NoFieldValue:
 _no_field_value = _NoFieldValue()
 
 
+def _to_primitive_placeholder(v: Any) -> Any:
+    if v is _no_field_value or v is no_default:
+        return {}
+    return v
+
+
+def _restore_primitive_placeholder(v: Any) -> Any:
+    if type(v) is dict:
+        if v == {}:
+            return _no_field_value
+        else:
+            return v
+    else:
+        return v
+
+
 class SerializableSerializer(Serializer):
     """
     Leverage DictSerializer to perform serde.
     """
 
+    @classmethod
+    def _get_obj_field_count_key(cls, obj: Serializable, legacy: bool = False):
+        return f"FC_{obj._NAME_HASH if not legacy else obj._LEGACY_NAME_HASH}"
+
     @classmethod
     def _get_field_values(cls, obj: Serializable, fields):
         values = []
@@ -201,15 +258,25 @@ class SerializableSerializer(Serializer):
         else:
             primitive_vals = self._get_field_values(obj, obj._PRIMITIVE_FIELDS)
             # replace _no_field_value as {} to make them msgpack-serializable
-            primitive_vals = [
-                v if v is not _no_field_value else {} for v in primitive_vals
-            ]
+            primitive_vals = [_to_primitive_placeholder(v) for v in primitive_vals]
             if obj._cache_primitive_serial:
                 primitive_vals = msgpack.dumps(primitive_vals)
                 _primitive_serial_cache[obj] = primitive_vals
 
         compound_vals = self._get_field_values(obj, obj._NON_PRIMITIVE_FIELDS)
         cls_module = f"{type(obj).__module__}#{type(obj).__qualname__}"
+
+        field_count_key = self._get_obj_field_count_key(obj)
+        if not self.is_public_data_exist(context, field_count_key):
+            # store field distribution for current Serializable
+            counts = [
+                list(obj._CLS_TO_PRIMITIVE_FIELD_COUNT.items()),
+                list(obj._CLS_TO_NON_PRIMITIVE_FIELD_COUNT.items()),
+            ]
+            field_count_data = msgpack.dumps(counts)
+            self.put_public_data(
+                context, self._get_obj_field_count_key(obj), field_count_data
+            )
         return [cls_module, primitive_vals], [compound_vals], False
 
     @staticmethod
@@ -229,6 +296,92 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _set_field_values(
+        cls,
+        obj: Serializable,
+        values: List[Any],
+        client_cls_to_field_count: Optional[Dict[str, int]],
+        is_primitive: bool = True,
+    ):
+        obj_class = type(obj)
+        if is_primitive:
+            server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
+            server_fields = obj_class._PRIMITIVE_FIELDS
+        else:
+            server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
+            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+
+        legacy_to_new_hash = {
+            c._LEGACY_NAME_HASH: c._NAME_HASH
+            for c in obj_class.__mro__
+            if hasattr(c, "_NAME_HASH") and c._LEGACY_NAME_HASH != c._NAME_HASH
+        }
+
+        if client_cls_to_field_count:
+            field_num, server_field_num = 0, 0
+            for cls_hash, count in client_cls_to_field_count.items():
+                # cut values and fields given field distribution
+                # at client and server end
+                cls_fields = server_fields[server_field_num : field_num + count]
+                cls_values = values[field_num : field_num + count]
+                for field, value in zip(cls_fields, cls_values):
+                    if is_primitive:
+                        value = _restore_primitive_placeholder(value)
+                    if not is_primitive or value is not _no_field_value:
+                        cls._set_field_value(obj, field, value)
+                field_num += count
+                try:
+                    server_field_num += server_cls_to_field_count[cls_hash]
+                except KeyError:
+                    try:
+                        # todo remove this fallback when all
+                        #  versions below v1.0.0rc1 is eliminated
+                        server_field_num += server_cls_to_field_count[
+                            legacy_to_new_hash[cls_hash]
+                        ]
+                    except KeyError:
+                        # it is possible that certain type of field does not exist
+                        #  at server side
+                        pass
+        else:
+            # handle legacy serialization style, with all fields sorted by name
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            from .field import AnyField
+
+            if is_primitive:
+                new_field_attr = "_legacy_new_primitives"
+                deprecated_field_attr = "_legacy_deprecated_primitives"
+            else:
+                new_field_attr = "_legacy_new_non_primitives"
+                deprecated_field_attr = "_legacy_deprecated_non_primitives"
+
+            # remove fields added on later releases
+            new_names = set(getattr(obj_class, new_field_attr, None) or [])
+            server_fields = [f for f in server_fields if f.name not in new_names]
+
+            # fill fields deprecated on later releases
+            deprecated_fields = []
+            deprecated_names = set()
+            if hasattr(obj_class, deprecated_field_attr):
+                deprecated_names = set(getattr(obj_class, deprecated_field_attr))
+                for field_name in deprecated_names:
+                    field = AnyField(tag=field_name)
+                    field.name = field_name
+                    deprecated_fields.append(field)
+            server_fields = sorted(
+                server_fields + deprecated_fields, key=lambda f: f.name
+            )
+            for field, value in zip(server_fields, values):
+                if is_primitive:
+                    value = _restore_primitive_placeholder(value)
+                if not is_primitive or value is not _no_field_value:
+                    try:
+                        cls._set_field_value(obj, field, value)
+                    except AttributeError:  # pragma: no cover
+                        if field.name not in deprecated_names:
+                            raise
+
     def deserial(self, serialized: List, context: Dict, subs: List) -> Serializable:
         obj_class_name, primitives = serialized
         obj_class = load_type(obj_class_name, Serializable)
@@ -238,14 +391,26 @@ class SerializableSerializer(Serializer):
 
         obj = obj_class.__new__(obj_class)
 
-        if primitives:
-            for field, value in zip(obj_class._PRIMITIVE_FIELDS, primitives):
-                if value != {}:
-                    self._set_field_value(obj, field, value)
+        field_count_data = self.get_public_data(
+            context, self._get_obj_field_count_key(obj)
+        )
+        if field_count_data is None:
+            # todo remove this fallback when all
+            #  versions below v1.0.0rc1 is eliminated
+            field_count_data = self.get_public_data(
+                context, self._get_obj_field_count_key(obj, legacy=True)
+            )
+        if field_count_data is not None:
+            cls_to_prim_key, cls_to_non_prim_key = msgpack.loads(field_count_data)
+            cls_to_prim_key = dict(cls_to_prim_key)
+            cls_to_non_prim_key = dict(cls_to_non_prim_key)
+        else:
+            cls_to_prim_key, cls_to_non_prim_key = None, None
 
+        if primitives:
+            self._set_field_values(obj, primitives, cls_to_prim_key, True)
         if obj_class._NON_PRIMITIVE_FIELDS:
-            for field, value in zip(obj_class._NON_PRIMITIVE_FIELDS, subs[0]):
-                self._set_field_value(obj, field, value)
+            self._set_field_values(obj, subs[0], cls_to_non_prim_key, False)
         obj.__on_deserialize__()
         return obj
 

maxframe/serialization/serializables/field_type.py

@@ -46,6 +46,9 @@ class PrimitiveType(Enum):
     complex128 = 25
 
 
+_np_unicode = np.unicode_ if hasattr(np, "unicode_") else np.str_
+
+
 _primitive_type_to_valid_types = {
     PrimitiveType.bool: (bool, np.bool_),
     PrimitiveType.int8: (int, np.int8),
@@ -60,7 +63,7 @@ _primitive_type_to_valid_types = {
     PrimitiveType.float32: (float, np.float32),
     PrimitiveType.float64: (float, np.float64),
     PrimitiveType.bytes: (bytes, np.bytes_),
-    PrimitiveType.string: (str, np.unicode_),
+    PrimitiveType.string: (str, _np_unicode),
     PrimitiveType.complex64: (complex, np.complex64),
     PrimitiveType.complex128: (complex, np.complex128),
 }

maxframe/serialization/serializables/tests/test_serializable.py

@@ -21,6 +21,7 @@ import pytest
 
 from ....core import EntityData
 from ....lib.wrapped_pickle import switch_unpickle
+from ....utils import no_default
 from ... import deserialize, serialize
 from .. import (
     AnyField,
@@ -94,6 +95,11 @@ class MySimpleSerializable(Serializable):
     _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
 
 
+class MySubSerializable(MySimpleSerializable):
+    _m_int_val = Int64Field("m_int_val", default=250)
+    _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+
 class MySerializable(Serializable):
     _id = IdentityField("id")
     _any_val = AnyField("any_val")
@@ -138,10 +144,11 @@ class MySerializable(Serializable):
         oneof1_val=f"{__name__}.MySerializable",
         oneof2_val=MySimpleSerializable,
     )
+    _no_default_val = Float64Field("no_default_val", default=no_default)
 
 
 @pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
-@switch_unpickle
+@switch_unpickle(forbidden=False)
 def test_serializable(set_is_ci):
     my_serializable = MySerializable(
         _id="1",
@@ -165,7 +172,9 @@ def test_serializable(set_is_ci):
         _key_val=MyHasKey("aaa"),
         _ndarray_val=np.random.rand(4, 3),
         _datetime64_val=pd.Timestamp(123),
-        _timedelta64_val=pd.Timedelta(days=1),
+        _timedelta64_val=pd.Timedelta(
+            days=1, seconds=123, microseconds=345, nanoseconds=132
+        ),
         _datatype_val=np.dtype(np.int32),
         _index_val=pd.Index([1, 2]),
         _series_val=pd.Series(["a", "b"]),
@@ -180,6 +189,7 @@ def test_serializable(set_is_ci):
         _dict_val={"a": b"bytes_value"},
         _ref_val=MySerializable(),
         _oneof_val=MySerializable(_id="2"),
+        _no_default_val=no_default,
     )
 
     header, buffers = serialize(my_serializable)
@@ -187,12 +197,51 @@ def test_serializable(set_is_ci):
     _assert_serializable_eq(my_serializable, my_serializable2)
 
 
+@pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
+@switch_unpickle
+def test_compatible_serializable(set_is_ci):
+    global MySimpleSerializable, MySubSerializable
+
+    old_base, old_sub = MySimpleSerializable, MySubSerializable
+
+    try:
+        my_sub_serializable = MySubSerializable(
+            _id="id_val",
+            _list_val=["abcd", "wxyz"],
+            _ref_val=MyHasKey(),
+            _m_int_val=3412,
+            _m_str_val="dfgghj",
+        )
+        header, buffers = serialize(my_sub_serializable)
+
+        class MySimpleSerializable(Serializable):
+            _id = IdentityField("id")
+            _int_val = Int64Field("int_val", default=1000)
+            _list_val = ListField("list_val", default_factory=list)
+            _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
+            _dict_val = DictField("dict_val")
+
+        class MySubSerializable(MySimpleSerializable):
+            _m_int_val = Int64Field("m_int_val", default=250)
+            _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+        my_sub_serializable2 = deserialize(header, buffers)
+        assert type(my_sub_serializable) is not type(my_sub_serializable2)
+        _assert_serializable_eq(my_sub_serializable, my_sub_serializable2)
+    finally:
+        MySimpleSerializable, MySubSerializable = old_base, old_sub
+
+
 def _assert_serializable_eq(my_serializable, my_serializable2):
     for field_name, field in my_serializable._FIELDS.items():
-        if not hasattr(my_serializable, field.tag):
+        if not hasattr(my_serializable, field.name):
             continue
         expect_value = getattr(my_serializable, field_name)
-        actual_value = getattr(my_serializable2, field_name)
+        if expect_value is no_default:
+            assert not hasattr(my_serializable2, field.name)
+            continue
+        else:
+            actual_value = getattr(my_serializable2, field_name)
         if isinstance(expect_value, np.ndarray):
             np.testing.assert_array_equal(expect_value, actual_value)
         elif isinstance(expect_value, pd.DataFrame):
@@ -208,7 +257,7 @@ def _assert_serializable_eq(my_serializable, my_serializable2):
         elif callable(expect_value):
             assert expect_value(1) == actual_value(1)
         else:
-            assert expect_value == actual_value
+            assert expect_value == actual_value, f"Field {field_name}"
 
 
 @pytest.mark.parametrize("set_is_ci", [True], indirect=True)

maxframe/serialization/tests/test_serial.py

@@ -42,7 +42,7 @@ except ImportError:
 from ...lib.sparse import SparseMatrix
 from ...lib.wrapped_pickle import switch_unpickle
 from ...tests.utils import require_cudf, require_cupy
-from ...utils import lazy_import
+from ...utils import lazy_import, no_default
 from .. import (
     PickleContainer,
     RemoteException,
@@ -90,6 +90,7 @@ class CustomNamedTuple(NamedTuple):
         pd.Timedelta(102.234154131),
         {"abc": 5.6, "def": [3.4], "gh": None, "ijk": {}},
         OrderedDict([("abcd", 5.6)]),
+        no_default,
     ],
 )
 @switch_unpickle

maxframe/session.py

@@ -150,6 +150,10 @@ class AbstractSession(ABC):
     def session_id(self):
         return self._session_id
 
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
     def __eq__(self, other):
         return (
             isinstance(other, AbstractSession)
@@ -365,6 +369,15 @@ class AbstractAsyncSession(AbstractSession, metaclass=ABCMeta):
         Stop server.
         """
 
+    @abstractmethod
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get Logview address
+        Returns
+        -------
+            Logview address
+        """
+
     def close(self):
         asyncio.run(self.destroy())
 
@@ -549,6 +562,15 @@ class AbstractSyncSession(AbstractSession, metaclass=ABCMeta):
 
         return fetch(tileables, self, offsets=offsets, sizes=sizes)
 
+    @abstractmethod
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get logview address
+        Returns
+        -------
+            logview address
+        """
+
 
 def _delegate_to_isolated_session(func: Union[Callable, Coroutine]):
     if asyncio.iscoroutinefunction(func):
@@ -728,6 +750,11 @@ class AsyncSession(AbstractAsyncSession):
         await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(coro, self._loop))
         stop_isolation()
 
+    @implements(AbstractAsyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
 
 class ProgressBar:
     def __init__(self, show_progress):
@@ -949,6 +976,11 @@ class SyncSession(AbstractSyncSession):
     def get_cluster_versions(self) -> List[str]:
         pass  # pragma: no cover
 
+    @implements(AbstractSyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
     def destroy(self):
         coro = self._isolated_session.destroy()
         asyncio.run_coroutine_threadsafe(coro, self._loop).result()
@@ -1255,9 +1287,12 @@ def get_default_or_create(**kwargs):
         if session is None:
             # no session attached, try to create one
             warnings.warn(warning_msg)
-            session = new_session(
-                ODPS.from_global() or ODPS.from_environments(), **kwargs
+            odps_entry = (
+                kwargs.pop("odps_entry", None)
+                or ODPS.from_global()
+                or ODPS.from_environments()
             )
+            session = new_session(odps_entry=odps_entry, **kwargs)
             session.as_default()
     if isinstance(session, IsolatedAsyncSession):
         session = SyncSession.from_isolated_session(session)