maxframe 1.0.0rc4__cp310-cp310-win_amd64.whl → 1.1.1__cp310-cp310-win_amd64.whl

maxframe/learn/contrib/llm/models/dashscope.py

@@ -0,0 +1,73 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ..... import opcodes
+from .....serialization.serializables.core import Serializable
+from .....serialization.serializables.field import StringField
+from ..core import LLMOperator
+from ..multi_modal import MultiModalLLM
+from ..text import TextLLM
+
+
+class DashScopeLLMMixin(Serializable):
+    __slots__ = ()
+
+    _not_supported_params = {"stream", "incremental_output"}
+
+    def validate_params(self, params: Dict[str, Any]):
+        for k in params.keys():
+            if k in self._not_supported_params:
+                raise ValueError(f"{k} is not supported")
+
+
+class DashScopeTextLLM(TextLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        return DashScopeTextGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeMultiModalLLM(MultiModalLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        # TODO add precheck here
+        return DashScopeMultiModalGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeTextGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_TEXT_GENERATION
+
+
+class DashScopeMultiModalGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_MULTI_MODAL_GENERATION

maxframe/learn/contrib/llm/multi_modal.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class MultiModalLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: MultiModalLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, MultiModalLLM):
+        raise ValueError("model must be a MultiModalLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/learn/contrib/llm/text.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class TextLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: TextLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, TextLLM):
+        raise ValueError("model must be a TextLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/lib/sparse/tests/test_sparse.py

@@ -55,13 +55,13 @@ def test_sparse_creation():
     s = SparseNDArray(s1_data)
     assert s.ndim == 2
     assert isinstance(s, SparseMatrix)
-    assert_array_equal(s.toarray(), s1_data.A)
-    assert_array_equal(s.todense(), s1_data.A)
+    assert_array_equal(s.toarray(), s1_data.toarray())
+    assert_array_equal(s.todense(), s1_data.toarray())
 
     ss = pickle.loads(pickle.dumps(s))
     assert s == ss
-    assert_array_equal(ss.toarray(), s1_data.A)
-    assert_array_equal(ss.todense(), s1_data.A)
+    assert_array_equal(ss.toarray(), s1_data.toarray())
+    assert_array_equal(ss.todense(), s1_data.toarray())
 
     v = SparseNDArray(v1, shape=(3,))
     assert s.ndim
@@ -331,12 +331,12 @@ def test_sparse_dot():
 
     assert_array_equal(mls.dot(s1, v1_s), s1.dot(v1_data))
     assert_array_equal(mls.dot(s2, v1_s), s2.dot(v1_data))
-    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.A))
-    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.A))
+    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.toarray()))
+    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s), v1_data.dot(v1_data), almost=True)
     assert_array_equal(mls.dot(v2_s, v2_s), v2_data.dot(v2_data), almost=True)
 
-    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.A))
+    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s, sparse=False), v1_data.dot(v1_data))
 
 
@@ -390,7 +390,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -399,7 +399,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -408,7 +408,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3])
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3])
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -417,7 +417,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3], wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3], wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -427,7 +427,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -437,7 +437,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -447,7 +447,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -457,7 +457,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)

maxframe/opcodes.py

@@ -270,6 +270,7 @@ KURTOSIS = 351
 SEM = 352
 STR_CONCAT = 353
 MAD = 354
+MEDIAN = 355
 
 # tensor operator
 RESHAPE = 401
@@ -377,7 +378,6 @@ DROP_DUPLICATES = 728
 MELT = 729
 RENAME = 731
 INSERT = 732
-MAP_CHUNK = 733
 CARTESIAN_CHUNK = 734
 EXPLODE = 735
 REPLACE = 736
@@ -392,6 +392,10 @@ PIVOT_TABLE = 744
 
 FUSE = 801
 
+# LLM
+DASHSCOPE_TEXT_GENERATION = 810
+DASHSCOPE_MULTI_MODAL_GENERATION = 811
+
 # table like input for tensor
 TABLE_COO = 1003
 # store tensor as coo format
@@ -569,6 +573,8 @@ CHOLESKY_FUSE = 999988
 # MaxFrame-dedicated functions
 DATAFRAME_RESHUFFLE = 10001
 FLATMAP = 10002
+FLATJSON = 10003
+APPLY_CHUNK = 10004
 
 # MaxFrame internal operators
 DATAFRAME_PROJECTION_SAME_INDEX_MERGE = 100001

maxframe/serialization/core.pyx

@@ -37,7 +37,7 @@ from .._utils import NamedType
 from .._utils cimport TypeDispatcher
 
 from ..lib import wrapped_pickle as pickle
-from ..utils import arrow_type_from_str
+from ..utils import NoDefault, arrow_type_from_str, no_default
 
 try:
     from pandas import ArrowDtype
@@ -94,6 +94,7 @@ cdef:
     int COMPLEX_SERIALIZER = 12
     int SLICE_SERIALIZER = 13
     int REGEX_SERIALIZER = 14
+    int NO_DEFAULT_SERIALIZER = 15
     int PLACEHOLDER_SERIALIZER = 4096
 
 
@@ -803,6 +804,16 @@ cdef class RegexSerializer(Serializer):
         return re.compile((<bytes>(subs[0])).decode(), serialized[0])
 
 
+cdef class NoDefaultSerializer(Serializer):
+    serializer_id = NO_DEFAULT_SERIALIZER
+
+    cpdef serial(self, object obj, dict context):
+        return [], [], True
+
+    cpdef deserial(self, list obj, dict context, list subs):
+        return no_default
+
+
 cdef class Placeholder:
     """
     Placeholder object to reduce duplicated serialization
@@ -857,6 +868,7 @@ DtypeSerializer.register(ExtensionDtype)
 ComplexSerializer.register(complex)
 SliceSerializer.register(slice)
 RegexSerializer.register(re.Pattern)
+NoDefaultSerializer.register(NoDefault)
 PlaceholderSerializer.register(Placeholder)
 
 

maxframe/serialization/pandas.py

@@ -134,8 +134,10 @@ class ArraySerializer(Serializer):
                 data_parts = [obj.tolist()]
             else:
                 data_parts = [obj.to_numpy().tolist()]
-        else:
+        elif hasattr(obj, "_data"):
             data_parts = [getattr(obj, "_data")]
+        else:
+            data_parts = [getattr(obj, "_pa_array")]
         return [ser_type], [dtype] + data_parts, False
 
     def deserial(self, serialized: List, context: Dict, subs: List):
@@ -155,38 +157,66 @@ class PdTimestampSerializer(Serializer):
         else:
             zone_info = []
             ts = obj.to_pydatetime().timestamp()
-        return (
-            [int(ts), obj.microsecond, obj.nanosecond],
-            zone_info,
-            bool(zone_info),
-        )
+        elements = [int(ts), obj.microsecond, obj.nanosecond]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, zone_info, bool(zone_info)
 
     def deserial(self, serialized: List, context: Dict, subs: List):
         if subs:
-            val = pd.Timestamp.utcfromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
-            val = val.replace(tzinfo=datetime.timezone.utc).tz_convert(subs[0])
+            pydt = datetime.datetime.utcfromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+                "tzinfo": datetime.timezone.utc,
+            }
+            if len(serialized) > 3:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs).tz_convert(subs[0])
         else:
-            val = pd.Timestamp.fromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
+            pydt = datetime.datetime.fromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+            }
+            if len(serialized) >= 4:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs)
         return val
 
 
 class PdTimedeltaSerializer(Serializer):
     def serial(self, obj: pd.Timedelta, context: Dict):
-        return [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days], [], True
+        elements = [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, [], True
 
     def deserial(self, serialized: List, context: Dict, subs: List):
         days = 0 if len(serialized) < 4 else serialized[3]
+        unit = None if len(serialized) < 5 else serialized[4]
         seconds, microseconds, nanoseconds = serialized[:3]
-        return pd.Timedelta(
-            days=days,
-            seconds=seconds,
-            microseconds=microseconds,
-            nanoseconds=nanoseconds,
-        )
+        kwargs = {
+            "days": days,
+            "seconds": seconds,
+            "microseconds": microseconds,
+            "nanoseconds": nanoseconds,
+        }
+        if unit is not None:
+            kwargs["unit"] = unit
+        return pd.Timedelta(**kwargs)
 
 
 class NoDefaultSerializer(Serializer):

maxframe/serialization/serializables/core.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 
 import weakref
-from collections import defaultdict
+from collections import OrderedDict
 from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
 
 from ...lib.mmh3 import hash
+from ...utils import no_default
 from ..core import Placeholder, Serializer, buffered, load_type
 from .field import Field
 from .field_type import DictType, ListType, PrimitiveFieldType, TupleType
@@ -97,14 +98,18 @@ class SerializableMeta(type):
                 non_primitive_fields.append(v)
 
         # count number of fields for every base class
-        cls_to_primitive_field_count = defaultdict(lambda: 0)
-        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_primitive_field_count = OrderedDict()
+        cls_to_non_primitive_field_count = OrderedDict()
         for field_name in field_order:
             cls_hash = field_to_cls_hash[field_name]
             if field_name in primitive_field_names:
-                cls_to_primitive_field_count[cls_hash] += 1
+                cls_to_primitive_field_count[cls_hash] = (
+                    cls_to_primitive_field_count.get(cls_hash, 0) + 1
+                )
             else:
-                cls_to_non_primitive_field_count[cls_hash] += 1
+                cls_to_non_primitive_field_count[cls_hash] = (
+                    cls_to_non_primitive_field_count.get(cls_hash, 0) + 1
+                )
 
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
@@ -119,9 +124,11 @@ class SerializableMeta(type):
         properties["_FIELD_ORDER"] = field_order
         properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
-        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
+            cls_to_primitive_field_count
+        )
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
-        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
             cls_to_non_primitive_field_count
         )
         properties["__slots__"] = tuple(slots)
@@ -211,6 +218,22 @@ class _NoFieldValue:
 _no_field_value = _NoFieldValue()
 
 
+def _to_primitive_placeholder(v: Any) -> Any:
+    if v is _no_field_value or v is no_default:
+        return {}
+    return v
+
+
+def _restore_primitive_placeholder(v: Any) -> Any:
+    if type(v) is dict:
+        if v == {}:
+            return _no_field_value
+        else:
+            return v
+    else:
+        return v
+
+
 class SerializableSerializer(Serializer):
     """
     Leverage DictSerializer to perform serde.
@@ -241,9 +264,7 @@ class SerializableSerializer(Serializer):
         else:
             primitive_vals = self._get_field_values(obj, obj._PRIMITIVE_FIELDS)
             # replace _no_field_value as {} to make them msgpack-serializable
-            primitive_vals = [
-                v if v is not _no_field_value else {} for v in primitive_vals
-            ]
+            primitive_vals = [_to_primitive_placeholder(v) for v in primitive_vals]
             if obj._cache_primitive_serial:
                 primitive_vals = msgpack.dumps(primitive_vals)
                 _primitive_serial_cache[obj] = primitive_vals
@@ -281,21 +302,51 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _prune_server_fields(
+        cls,
+        client_cls_to_field_count: Optional[Dict[int, int]],
+        server_cls_to_field_count: Dict[int, int],
+        server_fields: list,
+    ) -> list:
+        if not client_cls_to_field_count:  # pragma: no cover
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            return server_fields
+        if set(client_cls_to_field_count.keys()) == set(
+            server_cls_to_field_count.keys()
+        ):
+            return server_fields
+        ret_server_fields = []
+        server_pos = 0
+        for cls_hash, count in server_cls_to_field_count.items():
+            if cls_hash in client_cls_to_field_count:
+                ret_server_fields.extend(server_fields[server_pos : server_pos + count])
+            server_pos += count
+        return ret_server_fields
+
     @classmethod
     def _set_field_values(
         cls,
         obj: Serializable,
         values: List[Any],
-        client_cls_to_field_count: Optional[Dict[str, int]],
+        client_cls_to_field_count: Optional[Dict[int, int]],
         is_primitive: bool = True,
     ):
         obj_class = type(obj)
         if is_primitive:
             server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._PRIMITIVE_FIELDS,
+            )
         else:
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._NON_PRIMITIVE_FIELDS,
+            )
 
         legacy_to_new_hash = {
             c._LEGACY_NAME_HASH: c._NAME_HASH
@@ -311,7 +362,9 @@ class SerializableSerializer(Serializer):
                 cls_fields = server_fields[server_field_num : field_num + count]
                 cls_values = values[field_num : field_num + count]
                 for field, value in zip(cls_fields, cls_values):
-                    if not is_primitive or value != {}:
+                    if is_primitive:
+                        value = _restore_primitive_placeholder(value)
+                    if not is_primitive or value is not _no_field_value:
                         cls._set_field_value(obj, field, value)
                 field_num += count
                 try:
@@ -356,7 +409,9 @@ class SerializableSerializer(Serializer):
                 server_fields + deprecated_fields, key=lambda f: f.name
             )
             for field, value in zip(server_fields, values):
-                if not is_primitive or value != {}:
+                if is_primitive:
+                    value = _restore_primitive_placeholder(value)
+                if not is_primitive or value is not _no_field_value:
                     try:
                         cls._set_field_value(obj, field, value)
                     except AttributeError:  # pragma: no cover

maxframe/serialization/serializables/field_type.py

@@ -46,6 +46,9 @@ class PrimitiveType(Enum):
     complex128 = 25
 
 
+_np_unicode = np.unicode_ if hasattr(np, "unicode_") else np.str_
+
+
 _primitive_type_to_valid_types = {
     PrimitiveType.bool: (bool, np.bool_),
     PrimitiveType.int8: (int, np.int8),
@@ -60,7 +63,7 @@ _primitive_type_to_valid_types = {
     PrimitiveType.float32: (float, np.float32),
     PrimitiveType.float64: (float, np.float64),
     PrimitiveType.bytes: (bytes, np.bytes_),
-    PrimitiveType.string: (str, np.unicode_),
+    PrimitiveType.string: (str, _np_unicode),
     PrimitiveType.complex64: (complex, np.complex64),
     PrimitiveType.complex128: (complex, np.complex128),
 }

maxframe/serialization/serializables/tests/test_serializable.py

@@ -21,6 +21,7 @@ import pytest
 
 from ....core import EntityData
 from ....lib.wrapped_pickle import switch_unpickle
+from ....utils import no_default
 from ... import deserialize, serialize
 from .. import (
     AnyField,
@@ -143,6 +144,7 @@ class MySerializable(Serializable):
         oneof1_val=f"{__name__}.MySerializable",
         oneof2_val=MySimpleSerializable,
     )
+    _no_default_val = Float64Field("no_default_val", default=no_default)
 
 
 @pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
@@ -187,6 +189,7 @@ def test_serializable(set_is_ci):
         _dict_val={"a": b"bytes_value"},
         _ref_val=MySerializable(),
         _oneof_val=MySerializable(_id="2"),
+        _no_default_val=no_default,
     )
 
     header, buffers = serialize(my_serializable)
@@ -218,7 +221,10 @@ def test_compatible_serializable(set_is_ci):
             _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
             _dict_val = DictField("dict_val")
 
-        class MySubSerializable(MySimpleSerializable):
+        class MyMidSerializable(MySimpleSerializable):
+            _i_bool_val = Int64Field("i_bool_val", default=True)
+
+        class MySubSerializable(MyMidSerializable):
             _m_int_val = Int64Field("m_int_val", default=250)
             _m_str_val = StringField("m_str_val", default="SUB_STR")
 
@@ -234,7 +240,11 @@ def _assert_serializable_eq(my_serializable, my_serializable2):
         if not hasattr(my_serializable, field.name):
             continue
         expect_value = getattr(my_serializable, field_name)
-        actual_value = getattr(my_serializable2, field_name)
+        if expect_value is no_default:
+            assert not hasattr(my_serializable2, field.name)
+            continue
+        else:
+            actual_value = getattr(my_serializable2, field_name)
         if isinstance(expect_value, np.ndarray):
             np.testing.assert_array_equal(expect_value, actual_value)
         elif isinstance(expect_value, pd.DataFrame):