jamcodec 0.1.8__tar.gz → 0.1.9__tar.gz

{jamcodec-0.1.8 → jamcodec-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jamcodec
-Version: 0.1.8
+Version: 0.1.9
 Summary: Python JAM Codec Library
 Author-email: JAMdot Technologies <dev@jamdot.tech>
 Project-URL: Homepage, https://github.com/JAMdotTech/py-jam-codec

jamcodec-0.1.9/jamcodec/encoding.py

@@ -0,0 +1,383 @@
+"""Prepare standard model values for binary encoding without codec wrappers."""
+
+import dataclasses
+import struct
+from types import GetSetDescriptorType
+
+from jamcodec import types
+from jamcodec.base import JamBytes, JamCodecType
+
+
+_WRAPPERS = {
+    types.UnsignedInteger: JamCodecType,
+    types.SignedInteger: JamCodecType,
+    types.Float: JamCodecType,
+    types.Bool: JamCodecType,
+    types.NullType: JamCodecType,
+    types.CompactDef: JamCodecType,
+    types.BitVec: JamCodecType,
+    types.BitArray: JamCodecType,
+    types.BytesDef: JamCodecType,
+    types.StringDef: JamCodecType,
+    types.VarInt64Def: JamCodecType,
+    types.HashDef: types.HashDefObject,
+    types.Struct: types.StructObject,
+    types.Tuple: JamCodecType,
+    types.Enum: JamCodecType,
+    types.Option: JamCodecType,
+    types.Vec: types.VecType,
+    types.BoundedVec: types.VecType,
+    types.Array: types.ArrayObject,
+    types.Map: JamCodecType,
+}
+_TYPE_METHODS = {
+    cls: (cls.new, cls.deserialize, cls.serialize, cls.encode,
+          getattr(cls, '_check_count', None))
+    for cls in _WRAPPERS
+}
+_WRAPPER_METHODS = {
+    cls: (cls.__init__, cls.deserialize, cls.serialize, cls.encode,
+          cls.__getattribute__, cls.__setattr__, cls.value)
+    for cls in set(_WRAPPERS.values())
+}
+_HOOK_NAMES = frozenset(('new', 'deserialize', 'serialize', 'encode', '_check_count'))
+_PRIMITIVES = frozenset(_WRAPPERS) - {
+    types.Struct, types.Tuple, types.Enum, types.Option,
+    types.Vec, types.BoundedVec, types.Array, types.Map,
+}
+_INTEGERS = frozenset((types.UnsignedInteger, types.SignedInteger, types.CompactDef, types.VarInt64Def))
+_MODEL_HOOKS = frozenset(('__getattr__', '__bool__', '__len__'))
+
+
+class _UseWrappers(Exception):
+    pass
+
+
+class _Encoder:
+    def __init__(self):
+        self.checked = set()
+        self.checked_types = set()
+        self.checked_wrappers = set()
+        self.model_fields = {}
+        self.active = set()
+        self.data = bytearray()
+
+    def check(self, codec):
+        cls = type(codec)
+        if id(codec) in self.checked:
+            return cls
+        wrapper = _WRAPPERS.get(cls)
+        if (wrapper is None or codec.scale_type_cls is not wrapper
+                or not _HOOK_NAMES.isdisjoint(codec.__dict__)):
+            raise _UseWrappers
+        if cls not in self.checked_types:
+            if (cls.new, cls.deserialize, cls.serialize, cls.encode,
+                    getattr(cls, '_check_count', None)) != _TYPE_METHODS[cls]:
+                raise _UseWrappers
+            self.checked_types.add(cls)
+        if wrapper not in self.checked_wrappers:
+            if (wrapper.__init__, wrapper.deserialize, wrapper.serialize, wrapper.encode,
+                    wrapper.__getattribute__, wrapper.__setattr__, wrapper.value) != _WRAPPER_METHODS[wrapper]:
+                raise _UseWrappers
+            self.checked_wrappers.add(wrapper)
+
+        if cls is types.Struct:
+            if type(codec.arguments) is not dict or any(type(name) is not str for name in codec.arguments):
+                raise _UseWrappers
+        elif cls is types.Enum:
+            if (type(codec.variants) is not dict or
+                    (codec._deserialize_type is not None and type(codec._deserialize_type) is not type)):
+                raise _UseWrappers
+            if any(type(name) is not str for name in codec.variants):
+                raise _UseWrappers
+        elif cls is types.Tuple:
+            if type(codec.values) not in (list, tuple):
+                raise _UseWrappers
+        elif cls in (types.Vec, types.BoundedVec, types.Array):
+            if cls is types.Array and type(codec.length) is not int:
+                raise _UseWrappers
+            if cls is types.BoundedVec and any(type(bound) is not int for bound in (
+                codec.minimum, codec.maximum, codec.multiple,
+            )):
+                raise _UseWrappers
+            if cls in (types.Vec, types.BoundedVec):
+                self.check(types.VarInt64)
+                if codec.type_def is types.U8:
+                    self.check(types.Bytes)
+        elif cls is types.Map:
+            self.check(types.VarInt64)
+        elif cls in (types.BytesDef, types.StringDef, types.BitVec):
+            self.check(types.VarInt64)
+        elif cls in (types.UnsignedInteger, types.SignedInteger, types.HashDef):
+            if (type(codec.byte_count) is not int
+                    or type(codec.bits) is not int):
+                raise _UseWrappers
+        elif cls is types.Float:
+            if type(codec.struct_format) is not str:
+                raise _UseWrappers
+        elif cls is types.BitArray:
+            if type(codec.length) is not int or type(codec.strict_decoding) is not bool:
+                raise _UseWrappers
+        self.checked.add(id(codec))
+        return cls
+
+    def fields(self, value):
+        """Read plain dataclass fields without invoking user-defined accessors."""
+        cls = type(value)
+        if type(cls) is not type or not dataclasses.is_dataclass(cls):
+            raise _UseWrappers
+        if cls not in self.model_fields:
+            if (cls.__getattribute__ is not object.__getattribute__ or
+                    issubclass(cls, JamCodecType)):
+                raise _UseWrappers
+            names = tuple(field.name for field in dataclasses.fields(cls))
+            remaining = set(names)
+            dictionary = None
+            for base in cls.__mro__:
+                attributes = vars(base)
+                if not _MODEL_HOOKS.isdisjoint(attributes):
+                    raise _UseWrappers
+                if dictionary is None and '__dict__' in attributes:
+                    dictionary = attributes['__dict__']
+                for name in remaining.intersection(attributes):
+                    if hasattr(type(attributes[name]), '__get__'):
+                        raise _UseWrappers
+                    remaining.remove(name)
+            if type(dictionary) is not GetSetDescriptorType:
+                raise _UseWrappers
+            self.model_fields[cls] = names
+        values = object.__getattribute__(value, '__dict__')
+        names = self.model_fields[cls]
+        if any(name not in values for name in names):
+            raise _UseWrappers
+        return values if len(values) == len(names) else {name: values[name] for name in names}
+
+    def plain_leaf(self, value):
+        cls = type(value)
+        if cls in (bytes, bytearray, int, float, bool, str, type(None)):
+            return
+        if value is types.Null and not any(
+            name in base.__dict__ for base in cls.__mro__ for name in ('__bool__', '__len__')
+        ):
+            return
+        if id(value) in self.active:
+            raise _UseWrappers
+        self.active.add(id(value))
+        try:
+            if cls in (list, tuple):
+                for item in value:
+                    self.plain_leaf(item)
+            elif cls is dict:
+                for key, item in value.items():
+                    self.plain_leaf(key)
+                    self.plain_leaf(item)
+            else:
+                raise _UseWrappers
+        finally:
+            self.active.remove(id(value))
+
+    def primitive(self, codec, cls, value, native_bytes):
+        value_type = type(value)
+        if cls in _INTEGERS:
+            if value_type is not int:
+                raise _UseWrappers
+            return value
+        if cls is types.Float:
+            if value_type is not float:
+                raise _UseWrappers
+            return value
+        if cls is types.Bool:
+            if value_type is not bool:
+                raise _UseWrappers
+            return value
+        if cls in (types.BytesDef, types.HashDef):
+            if value_type is bytearray:
+                value = bytes(value)
+                value_type = bytes
+            if value_type is not bytes:
+                raise _UseWrappers
+            if cls is types.HashDef and len(value) != codec.byte_count:
+                raise _UseWrappers
+            return value if native_bytes else codec.serialize(value)
+        if cls is types.StringDef:
+            if value_type is not str:
+                raise _UseWrappers
+            return value
+        # Less common leaves retain their existing normalization and validation.
+        self.plain_leaf(value)
+        return codec.serialize(codec.deserialize(value))
+
+    def length(self, count):
+        if count < 128:
+            self.data.append(count)
+        else:
+            self.data.extend(types.VarInt64.encode(count).data)
+
+    def prepare(self, codec, value, emit=True, native_bytes=True):
+        """Write plain inputs, or return serialized values for map sorting.
+
+        When emitting a container, only its None/non-None result matters to
+        Option; its bytes have already been appended to the output buffer.
+        """
+        cls = self.check(codec)
+        if cls in _PRIMITIVES:
+            result = self.primitive(codec, cls, value, native_bytes)
+            if emit:
+                if cls is types.HashDef:
+                    self.data.extend(result)
+                elif cls is types.UnsignedInteger:
+                    self.data.extend(result.to_bytes(codec.byte_count, 'little'))
+                elif cls is types.BytesDef:
+                    self.length(len(result))
+                    self.data.extend(result)
+                elif cls is types.Bool:
+                    self.data.append(result)
+                elif cls is types.VarInt64Def and 0 <= result < 128:
+                    self.data.append(result)
+                elif cls is not types.NullType:
+                    self.data.extend(codec.encode(result).data)
+            return result
+        if cls is types.Struct:
+            if type(value) is not dict:
+                value = self.fields(value)
+            elif any(type(name) is not str for name in value):
+                raise _UseWrappers
+            result = {} if not emit else True
+            for name, child in codec.arguments.items():
+                if name not in value:
+                    raise _UseWrappers
+                item = self.prepare(child, value[name], emit, native_bytes)
+                if not emit:
+                    result[name] = item
+            return result
+        if cls is types.Tuple:
+            if type(value) not in (tuple, list) or len(value) < len(codec.values):
+                raise _UseWrappers
+            result = [self.prepare(child, value[index], emit, native_bytes)
+                      for index, child in enumerate(codec.values)]
+            return True if emit else tuple(result)
+        if cls is types.Option:
+            start = len(self.data)
+            if emit:
+                self.data.append(0)
+            result = None if value is None else self.prepare(codec.some, value, emit, native_bytes)
+            if emit and result is not None:
+                self.data[start] = 1
+            elif emit:
+                # Option(Null) and nested Options normalize to None too.
+                del self.data[start + 1:]
+            return result
+        if cls is types.Enum:
+            if type(value) not in (dict, str):
+                fields = self.fields(value)
+                if codec._deserialize_type and isinstance(value, codec._deserialize_type):
+                    value = {name: item for name, item in fields.items() if item is not None}
+                else:
+                    raise _UseWrappers
+            if type(value) is str:
+                value = {value: None}
+            if len(value) != 1:
+                raise _UseWrappers
+            name, item = next(iter(value.items()))
+            if type(name) is not str:
+                raise _UseWrappers
+            for index, (variant, child) in enumerate(codec.variants.items()):
+                if type(child) is dict:
+                    index, child = child.get('id', index), child.get('type', child)
+                if variant == name:
+                    if type(index) is not int or not 0 <= index <= 255:
+                        raise _UseWrappers
+                    if emit:
+                        self.data.append(index)
+                    if child is None:
+                        return name
+                    item = self.prepare(child, item, emit, native_bytes)
+                    return True if emit else {name: item}
+            raise _UseWrappers
+        if cls in (types.Vec, types.BoundedVec, types.Array):
+            array = cls is types.Array
+            if type(value) is bytearray:
+                value = bytes(value)
+            if type(value) not in (list, tuple, bytes):
+                raise _UseWrappers
+            if array and (type(value) is tuple or len(value) != codec.length):
+                raise _UseWrappers
+            if type(value) is list and not value:
+                if cls is types.BoundedVec:
+                    codec._check_count(0)
+                if emit and not array:
+                    self.data.append(0)
+                return True if emit else []
+            if not array and codec.type_def is types.U8 and type(value) is not bytes:
+                raise _UseWrappers
+            if cls is types.BoundedVec:
+                codec._check_count(len(value))
+            if codec.type_def is types.U8 and type(value) is bytes:
+                result = value if native_bytes else codec.serialize(value)
+                if emit:
+                    if not array:
+                        self.length(len(value))
+                    self.data.extend(value)
+                return result
+            if type(value) not in (list, tuple) or (array and type(value) is not list):
+                raise _UseWrappers
+            child_class = self.check(codec.type_def)
+            if (emit and child_class is types.HashDef
+                    and all(type(item) is bytes and len(item) == codec.type_def.byte_count for item in value)):
+                if not array:
+                    self.length(len(value))
+                self.data.extend(b''.join(value))
+                return True
+            if emit and child_class in _INTEGERS and all(type(item) is int for item in value):
+                if child_class is types.UnsignedInteger:
+                    if not array:
+                        self.length(len(value))
+                    for item in value:
+                        self.data.extend(item.to_bytes(codec.type_def.byte_count, 'little'))
+                else:
+                    self.data.extend(codec.encode(list(value) if type(value) is tuple else value).data)
+                return True
+            if emit and not array:
+                self.length(len(value))
+            result = [self.prepare(codec.type_def, item, emit, native_bytes) for item in value]
+            return True if emit else result
+        if type(value) not in (dict, list, tuple):
+            raise _UseWrappers
+        if not value:
+            if emit:
+                self.data.append(0)
+            return []
+        # Maps sort serialized representations, including values for duplicate
+        # keys. Keep that representation and let the existing encoder sort it.
+        items = value.items() if type(value) is dict else value
+        result = []
+        for pair in items:
+            if type(pair) not in (tuple, list) or len(pair) != 2:
+                raise _UseWrappers
+            key, item = pair
+            result.append((self.prepare(codec.key_def, key, False, False),
+                           self.prepare(codec.value_def, item, False, False)))
+        if emit:
+            self.data.extend(codec.encode(result).data)
+        return result
+
+
+def encode_from_object(codec, value):
+    """Return binary data for standard inputs, or None to use normal wrappers."""
+    # Bypassing wrappers pays off for collection models. Small mixed records
+    # are cheaper through the existing normalization path.
+    if (type(codec) is not types.Struct or type(codec.arguments) is not dict
+            or not codec.arguments or not all(type(child) in (
+                types.Array, types.Vec, types.BoundedVec, types.Map,
+            ) for child in codec.arguments.values())):
+        return None
+    encoder = _Encoder()
+    try:
+        encoder.prepare(codec, value)
+    except (_UseWrappers, ValueError, TypeError, ArithmeticError, IndexError,
+            KeyError, AttributeError, RecursionError, struct.error):
+        # Probing uses only built-in hooks and plain inputs. On invalid or
+        # unsupported data, repeat the original path so normalization errors
+        # still take precedence over encode-time errors in earlier fields.
+        return None
+    return JamBytes(encoder.data)

{jamcodec-0.1.8 → jamcodec-0.1.9}/jamcodec/mixins.py

@@ -8,6 +8,7 @@ import json
 from jamcodec.base import JamCodecTypeDef, JamCodecType, JamBytes
 from jamcodec.types import Struct, StructObject, Option, Vec, Enum
 from jamcodec.decoding import decode_to_object
+from jamcodec.encoding import encode_from_object
 
 T = TypeVar('T')
 
@@ -69,7 +70,17 @@ class Serializable:
         return scale_type.to_serializable_obj()
 
     def to_jam_bytes(self) -> JamBytes:
-        scale_obj = self.to_codec_type()
+        to_codec_type = self.to_codec_type
+        if (getattr(to_codec_type, '__func__', None) is _DEFAULT_TO_CODEC_TYPE
+                and is_dataclass(self)):
+            codec = self.to_codec_def()
+            data = encode_from_object(codec, self)
+            if data is not None:
+                return data
+            scale_obj = codec.new()
+            scale_obj.deserialize(self)
+        else:
+            scale_obj = to_codec_type()
         if type(scale_obj) in (JamCodecType, StructObject):
             # The value is already normalized; only the encoded bytes are needed.
             return scale_obj.type_def.encode(scale_obj.value_serialized)
@@ -106,3 +117,6 @@ class Serializable:
         for field in dataclasses.fields(self):
             if getattr(self, field.name, None) is not None:
                 return field.name, getattr(self, field.name)
+
+
+_DEFAULT_TO_CODEC_TYPE = Serializable.to_codec_type

{jamcodec-0.1.8 → jamcodec-0.1.9}/jamcodec.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jamcodec
-Version: 0.1.8
+Version: 0.1.9
 Summary: Python JAM Codec Library
 Author-email: JAMdot Technologies <dev@jamdot.tech>
 Project-URL: Homepage, https://github.com/JAMdotTech/py-jam-codec

{jamcodec-0.1.8 → jamcodec-0.1.9}/jamcodec.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ jamcodec/__init__.py
 jamcodec/base.py
 jamcodec/constants.py
 jamcodec/decoding.py
+jamcodec/encoding.py
 jamcodec/exceptions.py
 jamcodec/mixins.py
 jamcodec/types.py
@@ -23,6 +24,7 @@ test/test_bytes.py
 test/test_collection_buffers.py
 test/test_compact.py
 test/test_direct_decode.py
+test/test_direct_encode.py
 test/test_enum.py
 test/test_float.py
 test/test_hashmap.py

jamcodec-0.1.9/test/test_direct_encode.py

@@ -0,0 +1,440 @@
+from dataclasses import dataclass, field
+from types import MethodType
+from unittest.mock import patch
+
+import pytest
+
+from jamcodec.base import JamBytes, JamCodecType
+from jamcodec.encoding import encode_from_object
+from jamcodec.exceptions import ScaleDeserializeException
+from jamcodec.mixins import Serializable
+from jamcodec.types import (
+    Array, BitArray, Bool, BoundedVec, Bytes, BytesDef, Enum, F32, F64, H256, I16, Map, Null,
+    Option, Struct, StructObject, Tuple, U8, U16, U32, UnsignedInteger, VarInt64, Vec,
+)
+
+
+def legacy_encode(value):
+    """The binary encoding path before direct normalization was introduced."""
+    wrapper = value.to_codec_type()
+    if type(wrapper) in (JamCodecType, StructObject):
+        return wrapper.type_def.encode(wrapper.value_serialized)
+    return wrapper.encode()
+
+
+def assert_matches_legacy(value):
+    assert_encoding_matches_legacy(value)
+    # Exercise the collection path too when the original model is a mixed
+    # record, which deliberately keeps the normal wrapper path at the root.
+    assert_encoding_matches_legacy(model(Vec(value.to_codec_def()), [value]))
+
+
+def assert_encoding_matches_legacy(value):
+    try:
+        expected = legacy_encode(value)
+    except Exception as error:
+        with pytest.raises(type(error)) as caught:
+            value.to_jam_bytes()
+        assert str(caught.value) == str(error)
+    else:
+        actual = value.to_jam_bytes()
+        assert actual.to_bytes() == expected.to_bytes()
+        assert actual.offset == 0
+        assert actual.length == expected.length
+
+
+@dataclass
+class Item(Serializable):
+    number: int = field(metadata={'codec': U16})
+    payload: bytes = field(metadata={'codec': Bytes})
+
+
+@dataclass
+class Choice(Serializable):
+    item: object = None
+    empty: object = None
+    _codec_type_def = Enum(item={'id': 17, 'type': Item.to_codec_def()}, empty={'id': 255, 'type': Null})
+
+
+@dataclass
+class Packet(Serializable):
+    records: list = field(metadata={'codec': BoundedVec(Item.to_codec_def(), 3, minimum=1)})
+    optional: object = field(metadata={'codec': Option(Item.to_codec_def())})
+    lookup: dict = field(metadata={'codec': Map(Tuple(H256, U16), Option(Bytes))})
+    pair: tuple = field(metadata={'codec': Tuple(U8, Array(U16, 2))})
+    fixed: bytes = field(metadata={'codec': Array(U8, 2)})
+    bits: list = field(metadata={'codec': BitArray(9)})
+    choice: Choice = field(metadata={'codec': Choice.to_codec_def()})
+
+
+@pytest.mark.parametrize('choice', [Choice(item=Item(7, b'item')), Choice(empty=Null)])
+def test_standard_nested_models_use_direct_normalization(choice):
+    value = Packet(
+        [Item(1, b'first'), Item(258, b'')], Item(65535, b'optional'),
+        {(bytes(32), 2): None, (b'\xff' * 32, 1): b'value'},
+        (3, [1, 258]), bytearray(b'ab'), [True] * 9, choice,
+    )
+    collection = model(Vec(Packet.to_codec_def()), [value])
+    assert encode_from_object(collection.to_codec_def(), collection) is not None
+    assert_matches_legacy(value)
+
+
+def model(codec, value):
+    @dataclass
+    class Value(Serializable):
+        item: object = field(metadata={'codec': codec})
+
+    return Value(value)
+
+
+@pytest.mark.parametrize('codec,value', [
+    (Array(U8, 2), [1, 255]),
+    (Array(U8, 2), 'é'),
+    (Array(U8, 2), '0x01ff'),
+    (Array(U16, 2), [1, 258]),
+    (Tuple(U16), 258),
+    (Tuple(U8, U16), [1, 258, 999]),
+    (Vec(U16), (1, 258)),
+    (Vec(U8), 'é'),
+    (BoundedVec(U8, 2, minimum=2, multiple=2), 'é'),
+    (Option(Null), Null),
+    (Option(Option(U16)), None),
+    (Bytes, [0, 255]),
+    (Bytes, '0x00ff'),
+    (H256, bytearray(range(32))),
+    (BitArray(9), [False] * 9),
+])
+def test_accepted_input_representations_match(codec, value):
+    assert_matches_legacy(model(codec, value))
+
+
+@pytest.mark.parametrize('codec,value', [
+    (U8, True), (U8, '1'), (U8, 256), (H256, b'short'), (H256, 7),
+    (Array(U8, 2), [256, 1]), (Array(U8, 2), [True, 1]),
+    (Array(U8, 2), b'x'), (Array(U16, 2), b'ab'), (Tuple(U8, U16), [1]),
+    (BoundedVec(U16, 4, minimum=2, multiple=2), [1, 2, 3]),
+    (BoundedVec(U8, 2, minimum=2), 'éa'), (Bytes, '0xgg'),
+    (BitArray(9), ['x']),
+])
+def test_validation_errors_match(codec, value):
+    assert_matches_legacy(model(codec, value))
+
+
+@pytest.mark.parametrize('first,second,first_codec,second_codec', [
+    (256, 7, U8, H256),
+    (256, '0xgg', U8, Bytes),
+    (b'short', 'not an integer', H256, U8),
+    ([256, 1], 'not an integer', Array(U8, 2), U8),
+])
+def test_all_normalization_finishes_before_binary_validation(first, second, first_codec, second_codec):
+    @dataclass
+    class Invalid(Serializable):
+        first: object = field(metadata={'codec': first_codec})
+        second: object = field(metadata={'codec': second_codec})
+
+    assert_matches_legacy(Invalid(first, second))
+
+
+@pytest.mark.parametrize('first,first_codec', [
+    ([256, 1], Vec(U8)),
+    ([256, 1], BoundedVec(U8, 1)),
+    (1e100, F32),
+    (-1, VarInt64),
+    ([(1, None), (1, b'x')], Map(U8, Option(Bytes))),
+])
+def test_early_binary_errors_defer_to_later_normalization_errors(first, first_codec):
+    @dataclass
+    class Invalid(Serializable):
+        first: object = field(metadata={'codec': first_codec})
+        second: object = field(metadata={'codec': H256})
+
+    assert_matches_legacy(Invalid(first, 7))
+
+
+@pytest.mark.parametrize('codec,value', [
+    (Vec(U8), b'abc'),
+    (Array(U8, 2), [1, 255]),
+    (Array(U16, 2), [1, 65535]),
+    (Array(H256, 2), [bytes(32), b'\xff' * 32]),
+    (BoundedVec(H256, 3), [bytes(32)]),
+    (Vec(I16), [-32768, 32767]),
+    (Vec(F32), [0.5, float('inf')]),
+    (Vec(F64), [0.5, float('nan')]),
+    (Tuple(Bool(), U32), (True, 2**32 - 1)),
+    (Option(Option(Null)), Null),
+])
+def test_native_values_use_the_direct_path(codec, value):
+    value = model(codec, value)
+    value = model(Vec(value.to_codec_def()), [value])
+    assert encode_from_object(value.to_codec_def(), value) == legacy_encode(value)
+
+
+@pytest.mark.parametrize('codec,value', [
+    (Map(Bytes, Bytes), [(b'key', b'z'), (b'key', b'a')]),
+    (Map(Option(Bytes), U8), [(None, 1), (b'x', 2)]),
+    (Map(U8, Option(H256)), [(1, None), (1, bytes(32))]),
+    (Map(Array(U8, 2), U8), [(bytearray(b'ab'), 1), (b'aa', 2)]),
+    (Map(U8, Enum(a=None, b=None)), [(1, 'b'), (1, 'a')]),
+    (Map(U8, U8), [(1, 2, 3)]),
+])
+def test_map_sorting_uses_the_legacy_serialized_representations(codec, value):
+    assert_matches_legacy(model(codec, value))
+
+
+def test_custom_codec_hooks_run_once_and_in_the_original_order():
+    events = []
+
+    class CustomByte(UnsignedInteger):
+        def deserialize(self, value):
+            events.append('deserialize')
+            return super().deserialize(value) + 1
+
+        def serialize(self, value):
+            events.append('serialize')
+            return value + 10
+
+        def encode(self, value):
+            events.append('encode')
+            return super().encode(value)
+
+    @dataclass
+    class Value(Serializable):
+        first: bytes = field(metadata={'codec': Bytes})
+        custom: int = field(metadata={'codec': CustomByte(8)})
+
+    value = Value(b'first', 2)
+    expected = legacy_encode(value)
+    events.clear()
+    assert value.to_jam_bytes() == expected
+    assert events == ['deserialize', 'serialize', 'encode']
+
+
+def test_custom_root_wrapper_keeps_post_encode_validation():
+    events = []
+
+    class Wrapper(StructObject):
+        def deserialize(self, value):
+            events.append(('deserialize', self._data is not None))
+            return super().deserialize(value)
+
+        def encode(self, value=None):
+            events.append(('encode',))
+            return JamBytes(b'\xaa' + super().encode(value).to_bytes())
+
+    @dataclass
+    class Value(Serializable):
+        number: int
+        _codec_type_def = Struct(number=U8).impl(Wrapper)
+
+    expected = legacy_encode(Value(7))
+    events.clear()
+    assert Value(7).to_jam_bytes() == expected
+    assert events == [('deserialize', False), ('encode',), ('deserialize', True)]
+
+
+def test_custom_to_codec_type_is_called_once():
+    calls = []
+
+    @dataclass
+    class Value(Serializable):
+        number: int = field(metadata={'codec': U8})
+
+        def to_codec_type(self):
+            calls.append('to_codec_type')
+            return super().to_codec_type()
+
+    assert Value(7).to_jam_bytes().to_bytes() == b'\x07'
+    assert calls == ['to_codec_type']
+
+
+def test_replaced_base_to_codec_type_is_called_once():
+    original = Serializable.to_codec_type
+    calls = []
+
+    def custom(self):
+        calls.append('to_codec_type')
+        return original(self)
+
+    with patch.object(Serializable, 'to_codec_type', custom):
+        assert model(Vec(U16), [1, 258]).to_jam_bytes() == JamBytes(b'\x02\x01\x00\x02\x01')
+    assert calls == ['to_codec_type']
+
+
+def test_mutable_definitions_and_wrapper_changes_are_not_cached_between_calls():
+    @dataclass
+    class Value(Serializable):
+        number: int
+        _codec_type_def = Struct(number=U8)
+
+    assert Value(7).to_jam_bytes().to_bytes() == b'\x07'
+    Value.to_codec_def().arguments['number'] = U16
+    assert Value(258).to_jam_bytes().to_bytes() == b'\x02\x01'
+
+    class Prefix(StructObject):
+        def encode(self, value=None):
+            return JamBytes(b'\xaa' + super().encode(value).to_bytes())
+
+    Value.to_codec_def().impl(Prefix)
+    assert Value(258).to_jam_bytes().to_bytes() == b'\xaa\x02\x01'
+
+
+@pytest.mark.parametrize('hook', ['deserialize', 'serialize', 'encode'])
+def test_builtin_instance_hooks_are_preserved(hook):
+    codec = UnsignedInteger(8)
+    value = model(codec, 2)
+    assert value.to_jam_bytes().to_bytes() == b'\x02'
+    calls = []
+    original = getattr(codec, hook)
+
+    def custom(self, item):
+        calls.append(hook)
+        return original(item)
+
+    setattr(codec, hook, MethodType(custom, codec))
+    expected = legacy_encode(value)
+    calls.clear()
+    assert value.to_jam_bytes() == expected
+    assert calls == [hook]
+
+
+def test_class_serialization_hooks_preserve_validation():
+    original = UnsignedInteger.serialize
+
+    def checked(self, value):
+        if value == 255:
+            raise ScaleDeserializeException('class validation')
+        return original(self, value)
+
+    with patch.object(UnsignedInteger, 'serialize', checked):
+        assert_matches_legacy(model(Vec(U16), [1, 255]))
+
+
+def test_byte_vectors_preserve_indirect_bytes_hooks():
+    def reject(self, value):
+        raise ScaleDeserializeException('indirect byte validation')
+
+    with patch.object(Bytes, 'serialize', MethodType(reject, Bytes)):
+        assert_matches_legacy(model(BoundedVec(U8, 4), b'ab'))
+
+
+def test_custom_iterable_is_consumed_once():
+    for use_generator in (False, True):
+        calls = []
+
+        class Values(list):
+            def __iter__(self):
+                calls.append('iterate')
+                return super().__iter__()
+
+        def generate():
+            calls.append('iterate')
+            yield 1
+            yield 258
+
+        value = model(Vec(U16), generate() if use_generator else Values([1, 258]))
+        assert value.to_jam_bytes().to_bytes() == b'\x02\x01\x00\x02\x01'
+        assert calls == ['iterate']
+
+
+def test_fallback_precedes_custom_attribute_access_and_schema_mutation():
+    def make():
+        calls = []
+        payload_codec = BytesDef()
+        encode = payload_codec.encode
+
+        @dataclass
+        class Nested(Serializable):
+            number: int = field(metadata={'codec': U8})
+
+        def read_number(self):
+            calls.append('get_number')
+
+            def require_hex(self, value):
+                assert type(value) is str
+                return encode(value)
+
+            payload_codec.encode = MethodType(require_hex, payload_codec)
+            return self.__dict__['number']
+
+        value = Nested(7)
+        Nested.number = property(read_number)
+
+        @dataclass
+        class Outer(Serializable):
+            payload: bytes = field(metadata={'codec': payload_codec})
+            nested: Nested = field(metadata={'codec': Nested.to_codec_def()})
+
+        return Outer(b'abc', value), calls
+
+    old, old_calls = make()
+    expected = legacy_encode(old)
+    new, new_calls = make()
+    assert new.to_jam_bytes() == expected
+    assert old_calls == new_calls == ['get_number']
+
+
+def test_custom_truthiness_is_not_invoked_during_fast_path_probing():
+    calls = []
+
+    class Integer(int):
+        def __bool__(self):
+            calls.append('bool')
+            return True
+
+    value = model(U8, Integer(7))
+    with pytest.raises(ValueError, match='Value must be an integer'):
+        value.to_jam_bytes()
+    assert calls == ['bool']
+
+
+def test_custom_truthiness_in_a_collection_is_invoked_once():
+    calls = []
+
+    class Integer(int):
+        def __bool__(self):
+            calls.append('bool')
+            return True
+
+    value = model(Vec(U16), [Integer(7)])
+    with pytest.raises(ValueError, match='Value must be an integer'):
+        value.to_jam_bytes()
+    assert calls == ['bool']
+
+
+def test_direct_encoding_does_not_mutate_input_collections_or_buffers():
+    @dataclass
+    class Collection(Serializable):
+        numbers: list = field(metadata={'codec': Array(U16, 2)})
+        hashes: list = field(metadata={'codec': Vec(H256)})
+        lookup: dict = field(metadata={'codec': Map(Bytes, Bytes)})
+
+    numbers = [1, 258]
+    digest = bytearray(range(32))
+    hashes = [digest]
+    lookup = {b'z': b'last', b'a': b'first'}
+    value = Collection(numbers, hashes, lookup)
+    first = value.to_jam_bytes()
+    assert first == legacy_encode(value)
+    assert value.numbers is numbers and numbers == [1, 258]
+    assert value.hashes is hashes and hashes[0] is digest and digest == bytearray(range(32))
+    assert value.lookup is lookup and list(lookup) == [b'z', b'a']
+    numbers[0] = 2
+    digest[0] = 255
+    assert first != value.to_jam_bytes()
+
+
+@pytest.mark.parametrize('kind', ['array', 'vec', 'bounded', 'map'])
+def test_empty_collections_do_not_construct_elements(kind):
+    class NoElement(UnsignedInteger):
+        def new(self, **kwargs):
+            raise AssertionError('An empty collection must not construct elements')
+
+    child = NoElement(8)
+    codec = {
+        'array': Array(child, 0),
+        'vec': Vec(child),
+        'bounded': BoundedVec(child, 0),
+        'map': Map(child, child),
+    }[kind]
+    value = model(codec, {} if kind == 'map' else [])
+    assert encode_from_object(value.to_codec_def(), value) == legacy_encode(value)