jamcodec 0.1.7__tar.gz → 0.1.9__tar.gz

{jamcodec-0.1.7 → jamcodec-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jamcodec
-Version: 0.1.7
+Version: 0.1.9
 Summary: Python JAM Codec Library
 Author-email: JAMdot Technologies <dev@jamdot.tech>
 Project-URL: Homepage, https://github.com/JAMdotTech/py-jam-codec

jamcodec-0.1.9/jamcodec/decoding.py

@@ -0,0 +1,125 @@
+"""Binary model decoding without intermediate serialized values."""
+
+import enum
+
+from jamcodec.base import JamBytes, JamCodecType, JamCodecTypeDef
+from jamcodec import types
+from jamcodec.exceptions import ScaleDecodeException
+
+
+class _DecodedValue(JamCodecType):
+    # Existing model converters (notably Option) recognize JamCodecType values.
+    # Only these two fields are needed; models are built after the whole decode.
+    __slots__ = ('type_def', 'value_object')
+
+    def __init__(self, type_def, value):
+        self.type_def = type_def
+        self.value_object = value
+
+
+_WRAPPERS = {
+    types.UnsignedInteger: JamCodecType,
+    types.SignedInteger: JamCodecType,
+    types.Float: JamCodecType,
+    types.Bool: JamCodecType,
+    types.NullType: JamCodecType,
+    types.CompactDef: JamCodecType,
+    types.BitVec: JamCodecType,
+    types.BitArray: JamCodecType,
+    types.BytesDef: JamCodecType,
+    types.StringDef: JamCodecType,
+    types.VarInt64Def: JamCodecType,
+    types.HashDef: types.HashDefObject,
+    types.Struct: types.StructObject,
+    types.Tuple: JamCodecType,
+    types.Enum: JamCodecType,
+    types.Option: JamCodecType,
+    types.Vec: types.VecType,
+    types.BoundedVec: types.VecType,
+    types.Array: types.ArrayObject,
+    types.Map: JamCodecType,
+}
+_CONTAINERS = frozenset((
+    types.Struct, types.Tuple, types.Enum, types.Option,
+    types.Vec, types.BoundedVec, types.Array, types.Map,
+))
+
+# A custom hook must still see a normal codec wrapper, including its serialized
+# value and stream offsets. Do not cache plans: definitions remain mutable.
+_HOOK_NAMES = frozenset(('new', 'decode', 'serialize', 'to_serializable_obj', '_check_count'))
+_BUILTINS = {
+    cls: (
+        wrapper,
+        (cls.new, cls.decode, cls.serialize, cls.to_serializable_obj),
+        (wrapper.decode, wrapper.serialize, wrapper.to_serializable_obj),
+    )
+    for cls, wrapper in _WRAPPERS.items()
+}
+
+
+def _decode_value(codec, data):
+    cls = type(codec)
+    wrapper, type_methods, wrapper_methods = _BUILTINS.get(cls, (None, None, None))
+    if (
+        wrapper is None
+        or codec.scale_type_cls is not wrapper
+        or not _HOOK_NAMES.isdisjoint(codec.__dict__)
+        or (cls.new, cls.decode, cls.serialize, cls.to_serializable_obj) != type_methods
+        or (wrapper.decode, wrapper.serialize, wrapper.to_serializable_obj) != wrapper_methods
+        # Python enums expose their codec tuple rather than constructing a model.
+        or (cls is types.Enum and codec._deserialize_type is not None
+            and issubclass(codec._deserialize_type, enum.Enum))
+    ):
+        value = codec.new()
+        value.decode(data)
+        return value
+
+    if cls not in _CONTAINERS:
+        # Primitive decoders already enforce their wire-format validation.
+        value = codec.decode(data)
+    elif cls is types.Struct:
+        value = {name: _decode_value(child, data) for name, child in codec.arguments.items()}
+    elif cls is types.Tuple:
+        if len(codec.values) == 1:
+            value = _decode_value(codec.values[0], data)
+        else:
+            value = tuple(_decode_value(child, data) for child in codec.values)
+    elif cls is types.Enum:
+        index = int.from_bytes(data.get_next_bytes(1), byteorder='little')
+        for variant_index, (name, child) in enumerate(codec.variants.items()):
+            if type(child) is dict:
+                variant_index = child.get('id', variant_index)
+                child = child.get('type', child)
+            if index == variant_index:
+                value = (name, _decode_value(child, data) if child is not None else None)
+                break
+        else:
+            raise ScaleDecodeException(f"Index '{index}' not present in Enum type mapping")
+    elif cls is types.Option:
+        value = None if data.get_next_bytes(1) == b'\x00' else _decode_value(codec.some, data)
+    elif cls in (types.Vec, types.BoundedVec):
+        if codec.type_def is types.U8:
+            # Byte vectors also delegate serialization hooks to the Bytes codec.
+            value = codec.new()
+            value.decode(data)
+            return value
+        start = data.offset
+        count = types.VarInt64.decode(data)
+        if cls is types.BoundedVec:
+            if data.data[start:data.offset] != types.VarInt64.encode(count).data:
+                raise ValueError('non-canonical sequence length')
+            codec._check_count(count)
+        value = [_decode_value(codec.type_def, data) for _ in range(count)]
+    elif cls is types.Array:
+        value = (codec.decode(data) if codec.type_def is types.U8 else
+                 [_decode_value(codec.type_def, data) for _ in range(codec.length)])
+    elif cls is types.Map:
+        count = types.VarInt64.decode(data)
+        value = [(_decode_value(codec.key_def, data), _decode_value(codec.value_def, data))
+                 for _ in range(count)]
+    return _DecodedValue(codec, value)
+
+
+def decode_to_object(codec: JamCodecTypeDef, data: JamBytes):
+    """Decode a model while preserving the normal wrapper path for custom codecs."""
+    return _decode_value(codec, data).to_serializable_obj()

jamcodec-0.1.9/jamcodec/encoding.py

@@ -0,0 +1,383 @@
+"""Prepare standard model values for binary encoding without codec wrappers."""
+
+import dataclasses
+import struct
+from types import GetSetDescriptorType
+
+from jamcodec import types
+from jamcodec.base import JamBytes, JamCodecType
+
+
+_WRAPPERS = {
+    types.UnsignedInteger: JamCodecType,
+    types.SignedInteger: JamCodecType,
+    types.Float: JamCodecType,
+    types.Bool: JamCodecType,
+    types.NullType: JamCodecType,
+    types.CompactDef: JamCodecType,
+    types.BitVec: JamCodecType,
+    types.BitArray: JamCodecType,
+    types.BytesDef: JamCodecType,
+    types.StringDef: JamCodecType,
+    types.VarInt64Def: JamCodecType,
+    types.HashDef: types.HashDefObject,
+    types.Struct: types.StructObject,
+    types.Tuple: JamCodecType,
+    types.Enum: JamCodecType,
+    types.Option: JamCodecType,
+    types.Vec: types.VecType,
+    types.BoundedVec: types.VecType,
+    types.Array: types.ArrayObject,
+    types.Map: JamCodecType,
+}
+_TYPE_METHODS = {
+    cls: (cls.new, cls.deserialize, cls.serialize, cls.encode,
+          getattr(cls, '_check_count', None))
+    for cls in _WRAPPERS
+}
+_WRAPPER_METHODS = {
+    cls: (cls.__init__, cls.deserialize, cls.serialize, cls.encode,
+          cls.__getattribute__, cls.__setattr__, cls.value)
+    for cls in set(_WRAPPERS.values())
+}
+_HOOK_NAMES = frozenset(('new', 'deserialize', 'serialize', 'encode', '_check_count'))
+_PRIMITIVES = frozenset(_WRAPPERS) - {
+    types.Struct, types.Tuple, types.Enum, types.Option,
+    types.Vec, types.BoundedVec, types.Array, types.Map,
+}
+_INTEGERS = frozenset((types.UnsignedInteger, types.SignedInteger, types.CompactDef, types.VarInt64Def))
+_MODEL_HOOKS = frozenset(('__getattr__', '__bool__', '__len__'))
+
+
+class _UseWrappers(Exception):
+    pass
+
+
+class _Encoder:
+    def __init__(self):
+        self.checked = set()
+        self.checked_types = set()
+        self.checked_wrappers = set()
+        self.model_fields = {}
+        self.active = set()
+        self.data = bytearray()
+
+    def check(self, codec):
+        cls = type(codec)
+        if id(codec) in self.checked:
+            return cls
+        wrapper = _WRAPPERS.get(cls)
+        if (wrapper is None or codec.scale_type_cls is not wrapper
+                or not _HOOK_NAMES.isdisjoint(codec.__dict__)):
+            raise _UseWrappers
+        if cls not in self.checked_types:
+            if (cls.new, cls.deserialize, cls.serialize, cls.encode,
+                    getattr(cls, '_check_count', None)) != _TYPE_METHODS[cls]:
+                raise _UseWrappers
+            self.checked_types.add(cls)
+        if wrapper not in self.checked_wrappers:
+            if (wrapper.__init__, wrapper.deserialize, wrapper.serialize, wrapper.encode,
+                    wrapper.__getattribute__, wrapper.__setattr__, wrapper.value) != _WRAPPER_METHODS[wrapper]:
+                raise _UseWrappers
+            self.checked_wrappers.add(wrapper)
+
+        if cls is types.Struct:
+            if type(codec.arguments) is not dict or any(type(name) is not str for name in codec.arguments):
+                raise _UseWrappers
+        elif cls is types.Enum:
+            if (type(codec.variants) is not dict or
+                    (codec._deserialize_type is not None and type(codec._deserialize_type) is not type)):
+                raise _UseWrappers
+            if any(type(name) is not str for name in codec.variants):
+                raise _UseWrappers
+        elif cls is types.Tuple:
+            if type(codec.values) not in (list, tuple):
+                raise _UseWrappers
+        elif cls in (types.Vec, types.BoundedVec, types.Array):
+            if cls is types.Array and type(codec.length) is not int:
+                raise _UseWrappers
+            if cls is types.BoundedVec and any(type(bound) is not int for bound in (
+                codec.minimum, codec.maximum, codec.multiple,
+            )):
+                raise _UseWrappers
+            if cls in (types.Vec, types.BoundedVec):
+                self.check(types.VarInt64)
+                if codec.type_def is types.U8:
+                    self.check(types.Bytes)
+        elif cls is types.Map:
+            self.check(types.VarInt64)
+        elif cls in (types.BytesDef, types.StringDef, types.BitVec):
+            self.check(types.VarInt64)
+        elif cls in (types.UnsignedInteger, types.SignedInteger, types.HashDef):
+            if (type(codec.byte_count) is not int
+                    or type(codec.bits) is not int):
+                raise _UseWrappers
+        elif cls is types.Float:
+            if type(codec.struct_format) is not str:
+                raise _UseWrappers
+        elif cls is types.BitArray:
+            if type(codec.length) is not int or type(codec.strict_decoding) is not bool:
+                raise _UseWrappers
+        self.checked.add(id(codec))
+        return cls
+
+    def fields(self, value):
+        """Read plain dataclass fields without invoking user-defined accessors."""
+        cls = type(value)
+        if type(cls) is not type or not dataclasses.is_dataclass(cls):
+            raise _UseWrappers
+        if cls not in self.model_fields:
+            if (cls.__getattribute__ is not object.__getattribute__ or
+                    issubclass(cls, JamCodecType)):
+                raise _UseWrappers
+            names = tuple(field.name for field in dataclasses.fields(cls))
+            remaining = set(names)
+            dictionary = None
+            for base in cls.__mro__:
+                attributes = vars(base)
+                if not _MODEL_HOOKS.isdisjoint(attributes):
+                    raise _UseWrappers
+                if dictionary is None and '__dict__' in attributes:
+                    dictionary = attributes['__dict__']
+                for name in remaining.intersection(attributes):
+                    if hasattr(type(attributes[name]), '__get__'):
+                        raise _UseWrappers
+                    remaining.remove(name)
+            if type(dictionary) is not GetSetDescriptorType:
+                raise _UseWrappers
+            self.model_fields[cls] = names
+        values = object.__getattribute__(value, '__dict__')
+        names = self.model_fields[cls]
+        if any(name not in values for name in names):
+            raise _UseWrappers
+        return values if len(values) == len(names) else {name: values[name] for name in names}
+
+    def plain_leaf(self, value):
+        cls = type(value)
+        if cls in (bytes, bytearray, int, float, bool, str, type(None)):
+            return
+        if value is types.Null and not any(
+            name in base.__dict__ for base in cls.__mro__ for name in ('__bool__', '__len__')
+        ):
+            return
+        if id(value) in self.active:
+            raise _UseWrappers
+        self.active.add(id(value))
+        try:
+            if cls in (list, tuple):
+                for item in value:
+                    self.plain_leaf(item)
+            elif cls is dict:
+                for key, item in value.items():
+                    self.plain_leaf(key)
+                    self.plain_leaf(item)
+            else:
+                raise _UseWrappers
+        finally:
+            self.active.remove(id(value))
+
+    def primitive(self, codec, cls, value, native_bytes):
+        value_type = type(value)
+        if cls in _INTEGERS:
+            if value_type is not int:
+                raise _UseWrappers
+            return value
+        if cls is types.Float:
+            if value_type is not float:
+                raise _UseWrappers
+            return value
+        if cls is types.Bool:
+            if value_type is not bool:
+                raise _UseWrappers
+            return value
+        if cls in (types.BytesDef, types.HashDef):
+            if value_type is bytearray:
+                value = bytes(value)
+                value_type = bytes
+            if value_type is not bytes:
+                raise _UseWrappers
+            if cls is types.HashDef and len(value) != codec.byte_count:
+                raise _UseWrappers
+            return value if native_bytes else codec.serialize(value)
+        if cls is types.StringDef:
+            if value_type is not str:
+                raise _UseWrappers
+            return value
+        # Less common leaves retain their existing normalization and validation.
+        self.plain_leaf(value)
+        return codec.serialize(codec.deserialize(value))
+
+    def length(self, count):
+        if count < 128:
+            self.data.append(count)
+        else:
+            self.data.extend(types.VarInt64.encode(count).data)
+
+    def prepare(self, codec, value, emit=True, native_bytes=True):
+        """Write plain inputs, or return serialized values for map sorting.
+
+        When emitting a container, only its None/non-None result matters to
+        Option; its bytes have already been appended to the output buffer.
+        """
+        cls = self.check(codec)
+        if cls in _PRIMITIVES:
+            result = self.primitive(codec, cls, value, native_bytes)
+            if emit:
+                if cls is types.HashDef:
+                    self.data.extend(result)
+                elif cls is types.UnsignedInteger:
+                    self.data.extend(result.to_bytes(codec.byte_count, 'little'))
+                elif cls is types.BytesDef:
+                    self.length(len(result))
+                    self.data.extend(result)
+                elif cls is types.Bool:
+                    self.data.append(result)
+                elif cls is types.VarInt64Def and 0 <= result < 128:
+                    self.data.append(result)
+                elif cls is not types.NullType:
+                    self.data.extend(codec.encode(result).data)
+            return result
+        if cls is types.Struct:
+            if type(value) is not dict:
+                value = self.fields(value)
+            elif any(type(name) is not str for name in value):
+                raise _UseWrappers
+            result = {} if not emit else True
+            for name, child in codec.arguments.items():
+                if name not in value:
+                    raise _UseWrappers
+                item = self.prepare(child, value[name], emit, native_bytes)
+                if not emit:
+                    result[name] = item
+            return result
+        if cls is types.Tuple:
+            if type(value) not in (tuple, list) or len(value) < len(codec.values):
+                raise _UseWrappers
+            result = [self.prepare(child, value[index], emit, native_bytes)
+                      for index, child in enumerate(codec.values)]
+            return True if emit else tuple(result)
+        if cls is types.Option:
+            start = len(self.data)
+            if emit:
+                self.data.append(0)
+            result = None if value is None else self.prepare(codec.some, value, emit, native_bytes)
+            if emit and result is not None:
+                self.data[start] = 1
+            elif emit:
+                # Option(Null) and nested Options normalize to None too.
+                del self.data[start + 1:]
+            return result
+        if cls is types.Enum:
+            if type(value) not in (dict, str):
+                fields = self.fields(value)
+                if codec._deserialize_type and isinstance(value, codec._deserialize_type):
+                    value = {name: item for name, item in fields.items() if item is not None}
+                else:
+                    raise _UseWrappers
+            if type(value) is str:
+                value = {value: None}
+            if len(value) != 1:
+                raise _UseWrappers
+            name, item = next(iter(value.items()))
+            if type(name) is not str:
+                raise _UseWrappers
+            for index, (variant, child) in enumerate(codec.variants.items()):
+                if type(child) is dict:
+                    index, child = child.get('id', index), child.get('type', child)
+                if variant == name:
+                    if type(index) is not int or not 0 <= index <= 255:
+                        raise _UseWrappers
+                    if emit:
+                        self.data.append(index)
+                    if child is None:
+                        return name
+                    item = self.prepare(child, item, emit, native_bytes)
+                    return True if emit else {name: item}
+            raise _UseWrappers
+        if cls in (types.Vec, types.BoundedVec, types.Array):
+            array = cls is types.Array
+            if type(value) is bytearray:
+                value = bytes(value)
+            if type(value) not in (list, tuple, bytes):
+                raise _UseWrappers
+            if array and (type(value) is tuple or len(value) != codec.length):
+                raise _UseWrappers
+            if type(value) is list and not value:
+                if cls is types.BoundedVec:
+                    codec._check_count(0)
+                if emit and not array:
+                    self.data.append(0)
+                return True if emit else []
+            if not array and codec.type_def is types.U8 and type(value) is not bytes:
+                raise _UseWrappers
+            if cls is types.BoundedVec:
+                codec._check_count(len(value))
+            if codec.type_def is types.U8 and type(value) is bytes:
+                result = value if native_bytes else codec.serialize(value)
+                if emit:
+                    if not array:
+                        self.length(len(value))
+                    self.data.extend(value)
+                return result
+            if type(value) not in (list, tuple) or (array and type(value) is not list):
+                raise _UseWrappers
+            child_class = self.check(codec.type_def)
+            if (emit and child_class is types.HashDef
+                    and all(type(item) is bytes and len(item) == codec.type_def.byte_count for item in value)):
+                if not array:
+                    self.length(len(value))
+                self.data.extend(b''.join(value))
+                return True
+            if emit and child_class in _INTEGERS and all(type(item) is int for item in value):
+                if child_class is types.UnsignedInteger:
+                    if not array:
+                        self.length(len(value))
+                    for item in value:
+                        self.data.extend(item.to_bytes(codec.type_def.byte_count, 'little'))
+                else:
+                    self.data.extend(codec.encode(list(value) if type(value) is tuple else value).data)
+                return True
+            if emit and not array:
+                self.length(len(value))
+            result = [self.prepare(codec.type_def, item, emit, native_bytes) for item in value]
+            return True if emit else result
+        if type(value) not in (dict, list, tuple):
+            raise _UseWrappers
+        if not value:
+            if emit:
+                self.data.append(0)
+            return []
+        # Maps sort serialized representations, including values for duplicate
+        # keys. Keep that representation and let the existing encoder sort it.
+        items = value.items() if type(value) is dict else value
+        result = []
+        for pair in items:
+            if type(pair) not in (tuple, list) or len(pair) != 2:
+                raise _UseWrappers
+            key, item = pair
+            result.append((self.prepare(codec.key_def, key, False, False),
+                           self.prepare(codec.value_def, item, False, False)))
+        if emit:
+            self.data.extend(codec.encode(result).data)
+        return result
+
+
+def encode_from_object(codec, value):
+    """Return binary data for standard inputs, or None to use normal wrappers."""
+    # Bypassing wrappers pays off for collection models. Small mixed records
+    # are cheaper through the existing normalization path.
+    if (type(codec) is not types.Struct or type(codec.arguments) is not dict
+            or not codec.arguments or not all(type(child) in (
+                types.Array, types.Vec, types.BoundedVec, types.Map,
+            ) for child in codec.arguments.values())):
+        return None
+    encoder = _Encoder()
+    try:
+        encoder.prepare(codec, value)
+    except (_UseWrappers, ValueError, TypeError, ArithmeticError, IndexError,
+            KeyError, AttributeError, RecursionError, struct.error):
+        # Probing uses only built-in hooks and plain inputs. On invalid or
+        # unsupported data, repeat the original path so normalization errors
+        # still take precedence over encode-time errors in earlier fields.
+        return None
+    return JamBytes(encoder.data)

{jamcodec-0.1.7 → jamcodec-0.1.9}/jamcodec/mixins.py

@@ -7,6 +7,8 @@ import json
 
 from jamcodec.base import JamCodecTypeDef, JamCodecType, JamBytes
 from jamcodec.types import Struct, StructObject, Option, Vec, Enum
+from jamcodec.decoding import decode_to_object
+from jamcodec.encoding import encode_from_object
 
 T = TypeVar('T')
 
@@ -68,7 +70,17 @@ class Serializable:
         return scale_type.to_serializable_obj()
 
     def to_jam_bytes(self) -> JamBytes:
-        scale_obj = self.to_codec_type()
+        to_codec_type = self.to_codec_type
+        if (getattr(to_codec_type, '__func__', None) is _DEFAULT_TO_CODEC_TYPE
+                and is_dataclass(self)):
+            codec = self.to_codec_def()
+            data = encode_from_object(codec, self)
+            if data is not None:
+                return data
+            scale_obj = codec.new()
+            scale_obj.deserialize(self)
+        else:
+            scale_obj = to_codec_type()
         if type(scale_obj) in (JamCodecType, StructObject):
             # The value is already normalized; only the encoded bytes are needed.
             return scale_obj.type_def.encode(scale_obj.value_serialized)
@@ -77,7 +89,10 @@ class Serializable:
 
     @classmethod
     def from_jam_bytes(cls: Type[T], scale_bytes: JamBytes) -> T:
-        scale_obj = cls.to_codec_def().new()
+        codec = cls.to_codec_def()
+        if getattr(cls.from_codec_type, '__func__', None) is Serializable.from_codec_type.__func__:
+            return decode_to_object(codec, scale_bytes)
+        scale_obj = codec.new()
         scale_obj.decode(scale_bytes)
         return cls.from_codec_type(scale_obj)
 
@@ -102,3 +117,6 @@ class Serializable:
         for field in dataclasses.fields(self):
             if getattr(self, field.name, None) is not None:
                 return field.name, getattr(self, field.name)
+
+
+_DEFAULT_TO_CODEC_TYPE = Serializable.to_codec_type

{jamcodec-0.1.7 → jamcodec-0.1.9}/jamcodec.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jamcodec
-Version: 0.1.7
+Version: 0.1.9
 Summary: Python JAM Codec Library
 Author-email: JAMdot Technologies <dev@jamdot.tech>
 Project-URL: Homepage, https://github.com/JAMdotTech/py-jam-codec

{jamcodec-0.1.7 → jamcodec-0.1.9}/jamcodec.egg-info/SOURCES.txt

@@ -5,6 +5,8 @@ requirements.txt
 jamcodec/__init__.py
 jamcodec/base.py
 jamcodec/constants.py
+jamcodec/decoding.py
+jamcodec/encoding.py
 jamcodec/exceptions.py
 jamcodec/mixins.py
 jamcodec/types.py
@@ -21,6 +23,8 @@ test/test_bounded_vec.py
 test/test_bytes.py
 test/test_collection_buffers.py
 test/test_compact.py
+test/test_direct_decode.py
+test/test_direct_encode.py
 test/test_enum.py
 test/test_float.py
 test/test_hashmap.py