structo 0.0.2__tar.gz → 0.0.3__tar.gz

{structo-0.0.2 → structo-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: structo
-Version: 0.0.2
+Version: 0.0.3
 Summary: Structify
 Author-email: Ben Brady <benbradybusiness@gmail.com>
 License-Expression: MIT

{structo-0.0.2 → structo-0.0.3}/examples/custom_serializer.py

@@ -1,7 +1,8 @@
-from structo import Serializer, serialize_to_bytes, deserialize_from_bytes
+from structo import Serializer
 from dataclasses import dataclass
 import typing as t
 
+
 # We don't inherit from SerialiableObject
 # since we're using a custom serializer
 @dataclass
@@ -10,14 +11,9 @@ class Flags:
     flag_b: bool
     flag_c: bool
 
-class FlagsSerialiser(Serializer[Flags]):
-    # This is optional, but allows for size calculations
-    @staticmethod
-    def length(format):
-        return 1
 
-    @staticmethod
-    def write(buf, format, value):
+class FlagsSerialiser(Serializer[Flags]):
+    def write(self, buf, value):
         byte = 0
         if value.flag_a:
             byte += 1 << 0
@@ -28,8 +24,7 @@ class FlagsSerialiser(Serializer[Flags]):
 
         byte = buf.write(bytes([byte]))
 
-    @staticmethod
-    def read(buf, format):
+    def read(self, buf):
         byte = buf.read(1)[0]
         flag_a = ((byte >> 0) & 1) != 0
         flag_b = ((byte >> 1) & 1) != 0
@@ -40,7 +35,11 @@ class FlagsSerialiser(Serializer[Flags]):
             flag_c=flag_c,
         )
 
-type FlagsDatatype = t.Annotated[Flags, FlagsSerialiser]
+    def sizeof(self):
+        return 1
+
+
+type FlagsDatatype = t.Annotated[Flags, FlagsSerialiser()]
 
 
 value = Flags(
@@ -48,7 +47,6 @@ value = Flags(
     flag_b=True,
     flag_c=False,
 )
-output = serialize_to_bytes(flags_datatype, value)
+output = FlagsSerialiser().to_bytes(value)
 print(output)
-print(deserialize_from_bytes(flags_datatype, output))
-
+print(FlagsSerialiser().from_bytes(output))

{structo-0.0.2 → structo-0.0.3}/examples/iterable_serializer.py

@@ -1,42 +1,42 @@
+import typing as t
+from typing import Annotated
 from structo import (
     SerializableObject,
     Serializer,
-    write_serializable,
-    read_serializable,
     uint32_LE,
     String,
     List,
 )
 import random
-import typing as t
 from pathlib import Path
 
 
 class Post(SerializableObject):
-    id: uint32_LE
-    author: String[uint32_LE]
-    tags: List[uint32_LE, String[uint32_LE]]
+    id: Annotated[int, uint32_LE]
+    author: Annotated[str, String(uint32_LE)]
+    tags: Annotated[list[str], List(String(uint32_LE), uint32_LE)]
+
+CONTINUE_BYTE = bytes([255])
+NULL_TERMINATOR = bytes([0])
 
 
 class PostsSerialiser(Serializer[t.Iterable[Post]]):
-    @staticmethod
-    def write(buf, format, value):
+    def write(self, buf, value):
         for item in value:
-            buf.write(bytes([255]))
-            write_serializable(buf, Post, item)
+            buf.write(CONTINUE_BYTE)
+            item.write(buf)
 
-        buf.write(bytes([0]))
+        buf.write(NULL_TERMINATOR)
 
-    @staticmethod
-    def read(buf, format):
+    def read(self, buf):
         while True:
-            continue_byte = buf.read(1)[0]
-            if continue_byte == 0:
+            continue_byte = buf.read(1)
+            if continue_byte == NULL_TERMINATOR:
                 break
-            assert continue_byte == 255, "Continue byte was not 255"
+            assert continue_byte == CONTINUE_BYTE, "Continue byte was not 255"
 
             print("Loading...")  # to prove it's interspliced loading and yielding
-            yield read_serializable(buf, Post)
+            yield Post.read(buf)
 
 
 type PostsIterable = t.Annotated[t.Iterable[Post], PostsSerialiser()]
@@ -56,9 +56,9 @@ output = Path("output.raw")
 if not output.exists():
     with open(output, "wb") as f:
         posts = generate_posts()
-        write_serializable(f, PostsIterable, posts)
+        PostsSerialiser().write(f, posts)
 
 with open(output, "rb") as f:
-    iterable_posts = read_serializable(f, PostsIterable)
+    iterable_posts = PostsSerialiser().read(f)
     for post in iterable_posts:
         print(post)

structo-0.0.3/examples/wav.py

@@ -0,0 +1,39 @@
+from typing import Annotated
+from structo import uint16_LE, uint32_LE, SerializableObject, Literal
+
+
+class WavHeader(SerializableObject):
+    chunk_id: Annotated[bytes, Literal(b"RIFF")]
+    chunk_size: Annotated[int, uint32_LE]
+    format: Annotated[bytes, Literal(b"WAVE")]
+
+
+class ChunkHeader(SerializableObject):
+    id: Annotated[bytes, Literal(b"fmt ", b"data")]
+    size: Annotated[int, uint32_LE]
+
+
+class WavFormat(SerializableObject):
+    audio_format: Annotated[int, uint16_LE]
+    num_channels: Annotated[int, uint16_LE]
+    sample_rate: Annotated[int, uint32_LE]
+    byte_range: Annotated[int, uint32_LE]
+    block_align: Annotated[int, uint16_LE]
+    bits_per_sample: Annotated[int, uint16_LE]
+
+
+with open("example.wav", "rb") as f:
+    WavHeader.read(f)
+
+    format_header = ChunkHeader.read(f)
+    assert format_header.id == b'fmt '
+
+    format_data = f.read(format_header.size)
+    format = WavFormat.from_bytes(format_data)
+
+    data_header = ChunkHeader.read(f)
+    assert data_header.id == b"data"
+    wav_data = f.read(data_header.size)
+
+print(format)
+print(len(wav_data))

structo-0.0.3/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+testpaths =
+    tests
+python_functions = " *"

{structo-0.0.2 → structo-0.0.3}/structo/__init__.py

@@ -2,11 +2,9 @@
 Structify
 """
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"
 
-from .object import SerializableObject
-from .serialise import write_serializable, read_serializable
-from .serializer import Serializer, Format
+from .serializer import Serializer
 from .types import (
     uint64_BE,
     uint64_LE,
@@ -39,11 +37,11 @@ from .types import (
     List,
     String,
     Blob,
-    Literal
+    Literal,
+    ObjectSerializer,
 )
+from .object import SerializableObject
 from .utils import (
     StructoReader,
     StructifyWriter,
-    serialize_to_bytes,
-    deserialize_from_bytes,
 )

structo-0.0.3/structo/object.py

@@ -0,0 +1,58 @@
+import io
+import annotationlib
+import typing as t
+from dataclasses import dataclass
+
+from .serializer import Serializer
+from .serialise import get_serializer
+
+
+class SerializableObjectMeta(type):
+    def __new__(cls, name, bases, dct):
+        new_class = t.cast(type, super().__new__(cls, name, bases, dct))
+        if name == "SerializableObject":
+            return new_class
+
+        annotate = annotationlib.get_annotate_from_class_namespace(dct)
+        _annotations: dict[str, Serializer] = {}
+
+        if annotate:
+            attrs = annotate(annotationlib.Format.VALUE_WITH_FAKE_GLOBALS)
+            for key, value in attrs.items():
+                try:
+                    serializer = get_serializer(value)
+                    _annotations[key] = serializer
+                except Exception as e:
+                    raise ValueError(
+                        f"Invalid attribute defintion for {name}.{key}"
+                    ) from e
+
+        return t.cast(SerializableObject, dataclass(new_class))
+
+
+# This is very messed up since we
+@t.dataclass_transform()
+class SerializableObject(metaclass=SerializableObjectMeta):
+    @classmethod
+    def serializer(cls) -> Serializer[t.Self]:
+        from .types import ObjectSerializer
+
+        return ObjectSerializer(cls)
+
+    @classmethod
+    def sizeof(cls) -> int | None:
+        return cls.serializer().sizeof()
+
+    def write(self, buf: io.Writer):
+        return self.serializer().write(buf, self)
+
+    @classmethod
+    def read(cls, buf: io.Reader) -> t.Self:
+        return cls.serializer().read(buf)
+
+    def to_bytes(self) -> bytes:
+        return self.serializer().to_bytes(self)
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> t.Self:
+        return cls.serializer().from_bytes(data)

structo-0.0.3/structo/serialise.py

@@ -0,0 +1,41 @@
+import io
+import typing as t
+from .serializer import Serializer
+
+
+def get_serializer(format: type) -> Serializer:
+    from .serializer import Serializer
+    from .object import SerializableObject
+    from .types.object import ObjectSerializer
+
+    if t.get_origin(format) is t.Annotated:
+        args = t.get_args(format)
+        serializers = [arg for arg in args if isinstance(arg, Serializer)]
+        assert len(serializers) != 0, f"No serializers for {format} found"
+        assert len(serializers) == 1, f"More than one serializers for {format} found"
+        serializer = serializers[0]
+
+        return serializer
+
+    if issubclass(format, SerializableObject):
+        return ObjectSerializer(format)
+
+    # Nicely formatted errors:
+    if format == int:
+        raise ValueError(
+            f"No serializer for int, you need to use structo.int32, structo.int32 or similar instead"
+        )
+    if format == float:
+        raise ValueError(
+            f"No serializer for float, you need to use structo.float32 or structo.float64 instead"
+        )
+    if format == bytes:
+        raise ValueError(
+            f"No serializer for bytes, you need to use structo.Buffer or structo.Blob instead"
+        )
+    if format == list:
+        raise ValueError(
+            f"No serializer for list, you need to use structo.List or structo.Array instead"
+        )
+
+    raise NotImplementedError(f"No serializer found for {format}")

structo-0.0.3/structo/serializer.py

@@ -0,0 +1,20 @@
+import io
+
+
+class Serializer[T]:
+    def sizeof(self) -> int | None:
+        return None
+
+    def write(self, buf: io.Writer, value: T): ...
+
+    def read(self, buf: io.Reader) -> T: ...
+
+    def to_bytes(self, value: T) -> bytes:
+        buf = io.BytesIO()
+        self.write(buf, value)
+        buf.seek(0)
+        return buf.getvalue()
+
+    def from_bytes(self, data: bytes) -> T:
+        buf = io.BytesIO(data)
+        return self.read(buf)

{structo-0.0.2 → structo-0.0.3}/structo/types/__init__.py

@@ -26,6 +26,10 @@ from .primatives import (
     float32_LE,
     float32,
 )
-from .fixed import Array, Buffer
-from .dynamic import List, String, Blob
+from .buffer import Buffer
+from .array import Array
+from .blob import Blob
+from .list import List
+from .string import String
 from .literal import Literal
+from .object import ObjectSerializer

structo-0.0.3/structo/types/array.py

@@ -0,0 +1,34 @@
+from ..serializer import Serializer
+
+
+class Array[T](Serializer[list[T]]):
+    length: int
+    type: Serializer[T]
+
+    def __init__(self, length: int, type: Serializer[T]) -> None:
+        assert length > 0, "Array must be longer than 0"
+        self.length = length
+        self.type = type
+
+    def write(self, buf, value):
+        assert (
+            len(value) == self.length
+        ), f"expected array with {self.length} length, receieved {len(value)}"
+
+        for item in value:
+            self.type.write(buf, item)
+
+    def read(self, buf):
+        items = []
+        for _ in range(self.length):
+            items.append(self.type.read(buf))
+
+        return items
+
+    def sizeof(self):
+        element_length = self.type.sizeof()
+        if element_length is None:
+            return None
+        else:
+            return element_length * self.length
+

structo-0.0.3/structo/types/blob.py

@@ -0,0 +1,19 @@
+from ..serializer import Serializer
+
+
+class Blob(Serializer[bytes]):
+    "A set of arbitrary bytes, prefixed with it's length"
+
+    length_type: Serializer[int]
+
+    def __init__(self, length_type: Serializer[int]) -> None:
+        self.length_type = length_type
+
+    def write(self, buf, value):
+        self.length_type.write(buf, len(value))
+        buf.write(value)
+
+    def read(self, buf):
+        length = self.length_type.read(buf)
+        data = buf.read(length)
+        return data

structo-0.0.3/structo/types/buffer.py

@@ -0,0 +1,21 @@
+from ..serializer import Serializer
+
+
+class Buffer(Serializer[bytes]):
+    length: int
+
+    def __init__(self, length: int) -> None:
+        self.length = length
+
+    def write(self, buf, value):
+        assert (
+            len(value) == self.length
+        ), f"expected data with length {self.length}, received {len(value)}"
+        buf.write(value)
+
+    def read(self, buf):
+        data = buf.read(self.length)
+        return data
+
+    def sizeof(self):
+        return self.length

structo-0.0.3/structo/types/list.py

@@ -0,0 +1,27 @@
+from ..serializer import Serializer
+
+
+class List[T](Serializer[list[T]]):
+    "A list of items, prefixed with it's length"
+
+    value_type: Serializer[T]
+    length_type: Serializer[int]
+
+    def __init__(self, value_type: Serializer[T], length_type: Serializer[int]) -> None:
+        self.value_type = value_type
+        self.length_type = length_type
+
+    def write(self, buf, value):
+        length = len(value)
+        self.length_type.write(buf, length)
+        for item in value:
+            self.value_type.write(buf, item)
+
+    def read(self, buf):
+        length = self.length_type.read(buf)
+        values = []
+        for _ in range(length):
+            value = self.value_type.read(buf)
+            values.append(value)
+
+        return values

structo-0.0.3/structo/types/literal.py

@@ -0,0 +1,30 @@
+from ..serializer import Serializer
+
+
+class Literal(Serializer[bytes]):
+    values: list[bytes]
+    _length: int
+
+    def __init__(self, *values: bytes) -> None:
+        assert len(values) > 0, "foo"
+        length = len(values[0])
+
+        for value in values:
+            assert (
+                len(value) == length
+            ), f"All values in structo.Literal have to be the same length, expected {value} to be length {length}"
+
+        self._length = length
+        self.values = list(values)
+
+    def write(self, buf, value):
+        assert value in self.values, f"{value} not in {b", ".join(self.values)}"
+        buf.write(self.values)
+
+    def read(self, buf):
+        value = buf.read(self._length)
+        assert value in self.values, f"{value} not in {b", ".join(self.values)}"
+        return value
+
+    def sizeof(self):
+        return self._length

structo-0.0.3/structo/types/object.py

@@ -0,0 +1,51 @@
+import io
+import annotationlib
+
+from ..serializer import Serializer
+from ..object import SerializableObject
+from ..serialise import get_serializer
+
+
+class ObjectSerializer[T: SerializableObject](Serializer[T]):
+    _annotations: dict[str, Serializer] = {}
+    _type: type
+
+    def __init__(self, type: type[T]) -> None:
+        annotations: dict[str, Serializer] = {}
+        self._type = type
+
+        attrs = annotationlib.get_annotations(type)
+        for key, value in attrs.items():
+            try:
+                annotations[key] = get_serializer(value)
+            except Exception as e:
+                raise ValueError(
+                    f"Invalid attribute defintion for {type.__name__}.{key}"
+                ) from e
+
+        self._annotations = annotations
+
+    def sizeof(self) -> int | None:
+        total_size = 0
+        for serializer in self._annotations.values():
+            field_size = serializer.sizeof()
+            if field_size is None:
+                return None
+            else:
+                total_size += field_size
+
+        return total_size
+
+    def write(self, buf: io.Writer, value: T):
+        for field_key, field_format in self._annotations.items():
+            field_value = getattr(value, field_key)
+            field_format.write(buf, field_value)
+
+    def read(self, buf: io.Reader) -> T:
+        attrs = {}
+
+        for field_key, field_format in self._annotations.items():
+            field_value = field_format.read(buf)
+            attrs[field_key] = field_value
+
+        return self._type(**attrs)

structo-0.0.3/structo/types/primatives.py

@@ -0,0 +1,101 @@
+import typing as t
+import io
+import struct
+from ..serializer import Serializer
+
+
+def struct_serializer(sformat: str, bytes: int) -> Serializer:
+    class StructSerializer(Serializer):
+        def sizeof(self):
+            return bytes
+
+        def write(self, buf: io.Writer, value: int):
+            data = struct.pack(sformat, value)
+            return buf.write(data)
+
+        def read(self, buf: io.Reader):
+            data = buf.read(bytes)
+            return struct.unpack(sformat, data)[0]
+
+    return StructSerializer()
+
+
+uint64_BE: Serializer[int] = struct_serializer(">Q", bytes=8)
+"**unsigned 64bit integer - big endian**"
+
+uint64_LE: Serializer[int] = struct_serializer("<Q", bytes=8)
+"**unsigned 64bit integer - little endian**"
+
+uint64 = uint64_BE
+"**unsigned 64bit integer - big endian**"
+
+uint32_BE: Serializer[int] = struct_serializer(">I", bytes=4)
+"**unsigned 32bit integer - big endian**"
+
+uint32_LE: Serializer[int] = struct_serializer("<I", bytes=4)
+"**unsigned 32bit integer - little endian**"
+
+uint32 = uint32_BE
+"**unsigned 32bit integer - big endian**"
+
+uint16_BE: Serializer[int] = struct_serializer(">H", bytes=2)
+"**unsigned 16bit integer - big endian**"
+
+uint16_LE: Serializer[int] = struct_serializer("<H", bytes=2)
+"**unsigned 16bit integer - little endian**"
+
+uint16 = uint16_BE
+"**unsigned 16bit integer - big endian**"
+
+uint8: Serializer[int] = struct_serializer("B", bytes=1)
+"**unsigned 8bit integer**"
+
+int64_BE: Serializer[int] = struct_serializer(">q", bytes=8)
+"**signed 64bit integer - big endian**"
+
+int64_LE: Serializer[int] = struct_serializer("<q", bytes=8)
+"**signed 64bit integer - little endian**"
+
+int64 = int64_BE
+"**signed 64bit integer - big endian**"
+
+int32_BE: Serializer[int] = struct_serializer(">i", bytes=4)
+"**signed 32bit integer - big endian**"
+
+int32_LE: Serializer[int] = struct_serializer("<i", bytes=4)
+"**signed 32bit integer - little endian**"
+
+int32 = int32_BE
+"**signed 32bit integer - big endian**"
+
+int16_BE: Serializer[int] = struct_serializer(">h", bytes=2)
+"**signed 16bit integer - big endian**"
+
+int16_LE: Serializer[int] = struct_serializer("<h", bytes=2)
+"**signed 16bit integer - little endian**"
+
+int16 = int16_BE
+"**signed 16bit integer - big endian**"
+
+int8: Serializer[int] = struct_serializer("b", bytes=1)
+"**signed 8bit integer**"
+
+
+float64_BE: Serializer[float] = struct_serializer(">d", bytes=8)
+"**64bit float - big endian**"
+
+float64_LE: Serializer[float] = struct_serializer("<d", bytes=8)
+"**64bit float - little endian**"
+
+float64 = float64_BE
+"**64bit float - big endian**"
+
+float32_BE: Serializer[float] = struct_serializer(">f", bytes=4)
+"**32bit float - big endian**"
+
+float32_LE: Serializer[float] = struct_serializer("<f", bytes=4)
+"**32bit float - little endian**"
+
+float32 = float32_BE
+"**32bit float - big endian**"
+

structo-0.0.3/structo/types/string.py

@@ -0,0 +1,21 @@
+import typing as t
+from ..serializer import Serializer
+
+
+class String(Serializer[str]):
+    "A unicode string, prefixed with it's byte length"
+
+    length_type: Serializer[int]
+
+    def __init__(self, length_type: Serializer[int]) -> None:
+        self.length_type = length_type
+
+    def write(self, buf, value):
+        data = value.encode("utf-8")
+        self.length_type.write(buf, len(data))
+        buf.write(data)
+
+    def read(self, buf):
+        length = self.length_type.read(buf)
+        data = buf.read(length)
+        return data.decode("utf-8")