packbin 0.1.9__py3-none-any.whl

packbin/__init__.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from packbin._errors import ShortPacket, TrailingBytes, TypeMismatch, UnpackResult
+from packbin._nodes import (
+    be,
+    bits,
+    bool,
+    bytes,
+    dict,
+    eq,
+    flags,
+    flag_byte,
+    group,
+    i8,
+    i16,
+    i32,
+    i64,
+    f32,
+    f64,
+    list,
+    repeat,
+    sized,
+    u2,
+    u8,
+    u16,
+    u32,
+    u64,
+    utf8,
+    when,
+)
+from packbin._scheme import BinaryPacker, Scheme
+
+__all__ = [
+    "BinaryPacker",
+    "Scheme",
+    "ShortPacket",
+    "TrailingBytes",
+    "TypeMismatch",
+    "UnpackResult",
+    "u8",
+    "u16",
+    "u32",
+    "u64",
+    "i8",
+    "i16",
+    "i32",
+    "i64",
+    "f32",
+    "f64",
+    "bool",
+    "bytes",
+    "be",
+    "flags",
+    "flag_byte",
+    "eq",
+    "when",
+    "repeat",
+    "group",
+    "sized",
+    "u2",
+    "bits",
+    "utf8",
+    "list",
+    "dict",
+]

packbin/_errors.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True, slots=True)
+class ShortPacket:
+    field: str
+    needed: int
+    left: int
+
+
+@dataclass(frozen=True, slots=True)
+class TrailingBytes:
+    left: int
+
+
+@dataclass(frozen=True, slots=True)
+class TypeMismatch:
+    expected: int
+    actual: int
+
+
+@dataclass(frozen=True, slots=True)
+class UnpackResult(Generic[T]):
+    ok: bool
+    value: T | None = None
+    error: ShortPacket | TrailingBytes | TypeMismatch | None = None
+
+    @property
+    def field(self) -> str | None:
+        if isinstance(self.error, ShortPacket):
+            return self.error.field
+        return None
+
+    @property
+    def needed(self) -> int | None:
+        if isinstance(self.error, ShortPacket):
+            return self.error.needed
+        return None
+
+    @property
+    def left(self) -> int | None:
+        if isinstance(self.error, (ShortPacket, TrailingBytes)):
+            return self.error.left
+        return None

packbin/_nodes.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+
+Get = Callable[[Any], Any]
+Set = Callable[[Any, Any], None]
+_builtin_list = list
+
+
+class _Node:
+    pass
+
+
+@dataclass(slots=True)
+class _Scalar(_Node):
+    field_id: int
+    get: Get
+    set: Set
+    kind: str
+    size: int
+    fmt_le: str
+    fmt_be: str
+    signed: bool
+    min_v: int | float
+    max_v: int | float
+    big_endian: bool = False
+
+    def endian_fmt(self) -> str:
+        return self.fmt_be if self.big_endian else self.fmt_le
+
+
+@dataclass(slots=True)
+class _Bytes(_Node):
+    field_id: int
+    get: Get
+    set: Set
+    size: int
+
+
+@dataclass(slots=True)
+class _Bool(_Node):
+    field_id: int
+    get: Get
+    set: Set
+
+
+@dataclass(slots=True)
+class _FlagByte(_Node):
+    bits: list[_Node]
+
+    def bit(self, field: _Node) -> _FlagBit:
+        index = len(self.bits)
+        if index >= 8:
+            raise ValueError("flags already has 8 bits")
+        self.bits.append(field)
+        return _FlagBit(self, field, index)
+
+
+@dataclass(slots=True)
+class _FlagBit(_Node):
+    owner: _FlagByte
+    field: _Node
+    index: int
+
+
+@dataclass(slots=True)
+class _Flags(_Node):
+    fields: list[_Node]
+
+
+@dataclass(slots=True)
+class _Eq:
+    field_id: int
+    value: Any
+
+
+@dataclass(slots=True)
+class _When(_Node):
+    condition: _Eq
+    fields: list[_Node]
+
+
+@dataclass(slots=True)
+class _Repeat(_Node):
+    fields: list[_Node]
+
+
+@dataclass(slots=True)
+class _Group(_Node):
+    fields: list[_Node]
+
+
+@dataclass(slots=True)
+class _Sized(_Node):
+    field_id: int
+    get: Get
+    set: Set
+    count: int
+
+
+@dataclass(slots=True)
+class _U2Slot(_Node):
+    field_id: int
+    get: Get
+    set: Set
+
+
+@dataclass(slots=True)
+class _U2(_Node):
+    slots: list[_U2Slot]
+
+
+@dataclass(slots=True)
+class _Bits(_Node):
+    field_id: int
+    get: Get
+    set: Set
+    count: int
+
+
+@dataclass(slots=True)
+class _Utf8(_Node):
+    field_id: int
+    get: Get
+    set: Set
+
+
+@dataclass(slots=True)
+class _List(_Node):
+    get: Get
+    set: Set
+    element: _Node
+
+
+@dataclass(slots=True)
+class _Dict(_Node):
+    get: Get
+    set: Set
+    element: _Node
+
+
+def _scalar(
+    field_id: int,
+    get: Get,
+    set: Set,
+    kind: str,
+    size: int,
+    fmt: str,
+    signed: bool,
+    min_v: int | float,
+    max_v: int | float,
+) -> _Scalar:
+    return _Scalar(
+        field_id=field_id,
+        get=get,
+        set=set,
+        kind=kind,
+        size=size,
+        fmt_le="<" + fmt,
+        fmt_be=">" + fmt,
+        signed=signed,
+        min_v=min_v,
+        max_v=max_v,
+    )
+
+
+def u8(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "u8", 1, "B", False, 0, 0xFF)
+
+
+def u16(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "u16", 2, "H", False, 0, 0xFFFF)
+
+
+def u32(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "u32", 4, "I", False, 0, 0xFFFFFFFF)
+
+
+def u64(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "u64", 8, "Q", False, 0, 0xFFFFFFFFFFFFFFFF)
+
+
+def i8(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "i8", 1, "b", True, -0x80, 0x7F)
+
+
+def i16(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "i16", 2, "h", True, -0x8000, 0x7FFF)
+
+
+def i32(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "i32", 4, "i", True, -0x80000000, 0x7FFFFFFF)
+
+
+def i64(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "i64", 8, "q", True, -0x8000000000000000, 0x7FFFFFFFFFFFFFFF)
+
+
+def f32(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "f32", 4, "f", True, float("-inf"), float("inf"))
+
+
+def f64(field_id: int, get: Get, set: Set) -> _Scalar:
+    return _scalar(field_id, get, set, "f64", 8, "d", True, float("-inf"), float("inf"))
+
+
+def bool(field_id: int, get: Get, set: Set) -> _Bool:  # noqa: A001
+    return _Bool(field_id=field_id, get=get, set=set)
+
+
+def bytes(field_id: int, get: Get, set: Set, n: int) -> _Bytes:  # noqa: A001
+    if n < 0:
+        raise ValueError("bytes length must be >= 0")
+    return _Bytes(field_id=field_id, get=get, set=set, size=n)
+
+
+def be(field: _Scalar) -> _Scalar:
+    if not isinstance(field, _Scalar):
+        raise TypeError("be() expects a numeric field")
+    return _Scalar(
+        field_id=field.field_id,
+        get=field.get,
+        set=field.set,
+        kind=field.kind,
+        size=field.size,
+        fmt_le=field.fmt_le,
+        fmt_be=field.fmt_be,
+        signed=field.signed,
+        min_v=field.min_v,
+        max_v=field.max_v,
+        big_endian=True,
+    )
+
+
+def flags(*fields: _Node) -> _Flags:
+    return _Flags(fields=_builtin_list(fields))
+
+
+def flag_byte() -> _FlagByte:
+    return _FlagByte(bits=[])
+
+
+def eq(field_id: int, value: Any) -> _Eq:
+    return _Eq(field_id=field_id, value=value)
+
+
+def when(condition: _Eq, *fields: _Node) -> _When:
+    return _When(condition=condition, fields=_builtin_list(fields))
+
+
+def repeat(*fields: _Node) -> _Repeat:
+    return _Repeat(fields=_builtin_list(fields))
+
+
+def group(*fields: _Node) -> _Group:
+    return _Group(fields=_builtin_list(fields))
+
+
+def sized(field_id: int, get: Get, set: Set, count: int) -> _Sized:
+    return _Sized(field_id=field_id, get=get, set=set, count=count)
+
+
+def u2(*slots: _U2Slot | tuple[int, Get, Set]) -> _U2:
+    if not slots:
+        raise ValueError("u2 needs at least one slot")
+    out: list[_U2Slot] = []
+    for slot in slots:
+        if isinstance(slot, _U2Slot):
+            out.append(slot)
+        else:
+            field_id, get, set = slot
+            out.append(_U2Slot(field_id=field_id, get=get, set=set))
+    return _U2(slots=out)
+
+
+def bits(field_id: int, get: Get, set: Set, count: int) -> _Bits:
+    return _Bits(field_id=field_id, get=get, set=set, count=count)
+
+
+def utf8(field_id: int, get: Get, set: Set) -> _Utf8:
+    return _Utf8(field_id=field_id, get=get, set=set)
+
+
+def list(get: Get, set: Set, element: _Node) -> _List:  # noqa: A001
+    if isinstance(element, _Repeat):
+        raise ValueError("repeat is not a list element")
+    return _List(get=get, set=set, element=element)
+
+
+def dict(get: Get, set: Set, element: _Node) -> _Dict:  # noqa: A001
+    if isinstance(element, _Repeat):
+        raise ValueError("repeat is not a dictionary element")
+    return _Dict(get=get, set=set, element=element)
+
+
+def _validate_order(nodes: Sequence[_Node], next_id: int = 0) -> int:
+    for node in nodes:
+        if isinstance(node, (_Scalar, _Bytes, _Bool, _Utf8, _Sized, _Bits)):
+            if node.field_id != next_id:
+                raise ValueError(f"field id {node.field_id} is not the next order {next_id}")
+            next_id += 1
+        elif isinstance(node, _U2Slot):
+            if node.field_id != next_id:
+                raise ValueError(f"field id {node.field_id} is not the next order {next_id}")
+            next_id += 1
+        elif isinstance(node, _U2):
+            next_id = _validate_order(node.slots, next_id)
+        elif isinstance(node, _Flags):
+            next_id = _validate_order(node.fields, next_id)
+        elif isinstance(node, (_When, _Repeat, _Group)):
+            next_id = _validate_order(node.fields, next_id)
+        elif isinstance(node, _FlagByte):
+            next_id = _validate_order(node.bits, next_id)
+        elif isinstance(node, _FlagBit):
+            next_id = _validate_order([node.field], next_id)
+        elif isinstance(node, (_List, _Dict)):
+            _validate_order([node.element], 0)
+        else:
+            raise TypeError(f"unknown field node: {type(node)!r}")
+    return next_id

packbin/_pack.py

@@ -0,0 +1,264 @@
+from __future__ import annotations
+
+import struct
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from packbin._nodes import (
+    _Bits,
+    _Bool,
+    _Bytes,
+    _Dict,
+    _FlagBit,
+    _FlagByte,
+    _Flags,
+    _Group,
+    _List,
+    _Node,
+    _Repeat,
+    _Scalar,
+    _Sized,
+    _U2,
+    _Utf8,
+    _When,
+)
+
+_builtin_bytes = bytes
+_builtin_list = list
+_builtin_dict = dict
+
+
+def _utf8_payload(label: str, value: Any) -> bytes:
+    if not isinstance(value, str):
+        raise TypeError(f"{label}: expected str")
+    raw = value.encode("utf-8")
+    if len(raw) > 65535:
+        raise ValueError(f"{label}: utf-8 length {len(raw)}")
+    return raw
+
+
+def _present_value(value: Any) -> bool:
+    return value is not None
+
+
+def _bool_on(value: Any) -> bool:
+    return value is True
+
+
+def _group_on(row: Any, node: _Group) -> bool:
+    for child in node.fields:
+        if _child_on(row, child):
+            return True
+    return False
+
+
+def _child_on(row: Any, child: _Node) -> bool:
+    if isinstance(child, _Group):
+        return _group_on(row, child)
+    if isinstance(child, _Bool):
+        return _bool_on(child.get(row))
+    if isinstance(child, (_Scalar, _Bytes, _Utf8, _Sized, _Bits, _List, _Dict)):
+        return _present_value(child.get(row))
+    if isinstance(child, _FlagBit):
+        return _child_on(row, child.field)
+    return False
+
+
+def _write_u2(buf: bytearray, slots: Sequence[Any], row: Any, seen: dict[int, Any]) -> None:
+    nbytes = (len(slots) + 3) // 4
+    raw = bytearray(nbytes)
+    for i, slot in enumerate(slots):
+        value = slot.get(row)
+        seen[slot.field_id] = value
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0 or value > 3:
+            raise ValueError(f"{slot.field_id}: expected 2-bit int")
+        raw[i // 4] |= value << ((i % 4) * 2)
+    buf.extend(raw)
+
+
+def _write_bits(buf: bytearray, label: str, count: int, raw: Any) -> None:
+    if not isinstance(raw, _builtin_list) or len(raw) != count:
+        raise ValueError(f"{label}: expected {count} bits")
+    nbytes = (count + 7) // 8
+    packed = bytearray(nbytes)
+    for i, bit in enumerate(raw):
+        if bit not in (0, 1):
+            raise ValueError(f"{label}: expected 0 or 1")
+        packed[i // 8] |= int(bit) << (i % 8)
+    buf.extend(packed)
+
+
+def _require_int(field: _Scalar, value: Any) -> int | float:
+    if field.kind in ("f32", "f64"):
+        return float(value)
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise TypeError(f"{field.field_id}: expected int, got {type(value).__name__}")
+    if value < field.min_v or value > field.max_v:
+        raise OverflowError(f"{field.field_id}: {value} does not fit in {field.kind}")
+    return value
+
+
+def _write_scalar(buf: bytearray, field: _Scalar, value: Any) -> None:
+    packed = struct.pack(field.endian_fmt(), _require_int(field, value))
+    buf.extend(packed)
+
+
+def _is_leaf(node: _Node) -> bool:
+    return isinstance(node, (_Scalar, _Bytes, _Utf8, _Bool, _Sized, _Bits))
+
+
+def _pack_element(buf: bytearray, element: _Node, item: Any) -> None:
+    if isinstance(element, _List):
+        _pack_list_items(buf, element.element, item)
+    elif isinstance(element, _Dict):
+        _pack_dict_items(buf, element.element, item)
+    elif _is_leaf(element):
+        pack_nodes(buf, [element], item, {}, lambda _n: item)
+    else:
+        pack_nodes(buf, [element], item, {})
+
+
+def _pack_list_items(buf: bytearray, element: _Node, items: Any) -> None:
+    if not isinstance(items, _builtin_list):
+        raise TypeError("list: expected list")
+    if len(items) > 65535:
+        raise ValueError(f"list: length {len(items)}")
+    buf.append(len(items) & 0xFF)
+    buf.append((len(items) >> 8) & 0xFF)
+    for item in items:
+        _pack_element(buf, element, item)
+
+
+def _pack_dict_items(buf: bytearray, element: _Node, mapping: Any) -> None:
+    if not isinstance(mapping, _builtin_dict):
+        raise TypeError("dict: expected dict")
+    if len(mapping) > 65535:
+        raise ValueError(f"dict: length {len(mapping)}")
+    pairs = sorted(mapping.items(), key=lambda kv: kv[0].encode("utf-8"))
+    buf.append(len(pairs) & 0xFF)
+    buf.append((len(pairs) >> 8) & 0xFF)
+    for key, value in pairs:
+        raw = _utf8_payload("dict", key)
+        buf.append(len(raw) & 0xFF)
+        buf.append((len(raw) >> 8) & 0xFF)
+        buf.extend(raw)
+        _pack_element(buf, element, value)
+
+
+def pack_nodes(
+    buf: bytearray,
+    nodes: Sequence[_Node],
+    row: Any,
+    seen: dict[int, Any] | None = None,
+    get_value: Callable[[_Node], Any] | None = None,
+) -> None:
+    if seen is None:
+        seen = {}
+
+    def take(node: _Node) -> Any:
+        if get_value is not None:
+            return get_value(node)
+        return node.get(row)  # type: ignore[attr-defined]
+
+    for node in nodes:
+        if isinstance(node, _Scalar):
+            value = take(node)
+            if not _present_value(value) and get_value is None:
+                raise KeyError(f"missing field {node.field_id}")
+            seen[node.field_id] = value
+            _write_scalar(buf, node, value)
+        elif isinstance(node, _Bytes):
+            raw = take(node)
+            if not _present_value(raw) and get_value is None:
+                raise KeyError(f"missing field {node.field_id}")
+            seen[node.field_id] = raw
+            if not isinstance(raw, (_builtin_bytes, bytearray, memoryview)):
+                raise TypeError(f"{node.field_id}: expected bytes")
+            if len(raw) != node.size:
+                raise ValueError(f"{node.field_id}: expected {node.size} bytes, got {len(raw)}")
+            buf.extend(raw)
+        elif isinstance(node, _Bool):
+            seen[node.field_id] = take(node)
+        elif isinstance(node, _Flags):
+            flag = 0
+            for i, child in enumerate(node.fields):
+                if _child_on(row, child):
+                    flag |= 1 << i
+            buf.append(flag)
+            for i, child in enumerate(node.fields):
+                if flag & (1 << i):
+                    if isinstance(child, _Bool):
+                        seen[child.field_id] = child.get(row)
+                    elif isinstance(child, _Group):
+                        pack_nodes(buf, child.fields, row, seen, get_value)
+                    else:
+                        pack_nodes(buf, [child], row, seen, get_value)
+        elif isinstance(node, _Sized):
+            count = seen.get(node.count)
+            if count is None:
+                raise RuntimeError(f"{node.field_id}: count {node.count} is missing")
+            raw = take(node)
+            seen[node.field_id] = raw
+            if not isinstance(raw, (_builtin_bytes, bytearray, memoryview)):
+                raise TypeError(f"{node.field_id}: expected bytes")
+            if len(raw) != count:
+                raise ValueError(f"{node.field_id}: expected {count} bytes, got {len(raw)}")
+            buf.extend(raw)
+        elif isinstance(node, _U2):
+            _write_u2(buf, node.slots, row, seen)
+        elif isinstance(node, _Bits):
+            count = seen.get(node.count)
+            if count is None:
+                raise RuntimeError(f"{node.field_id}: count {node.count} is missing")
+            value = take(node)
+            seen[node.field_id] = value
+            _write_bits(buf, str(node.field_id), int(count), value)
+        elif isinstance(node, _FlagByte):
+            flag = 0
+            for i, child in enumerate(node.bits):
+                if _child_on(row, child):
+                    flag |= 1 << i
+            buf.append(flag)
+        elif isinstance(node, _FlagBit):
+            if _child_on(row, node.field):
+                pack_nodes(buf, [node.field], row, seen, get_value)
+        elif isinstance(node, _When):
+            if seen.get(node.condition.field_id) == node.condition.value:
+                pack_nodes(buf, node.fields, row, seen, get_value)
+        elif isinstance(node, _Repeat):
+            lengths = []
+            for child in node.fields:
+                val = child.get(row)  # type: ignore[attr-defined]
+                if val is None:
+                    lengths.append(0)
+                elif isinstance(val, _builtin_list):
+                    lengths.append(len(val))
+                else:
+                    lengths.append(1)
+            count = max(lengths) if lengths else 0
+            if any(n not in (0, count) for n in lengths):
+                raise ValueError("repeat fields must have equal lengths")
+            for i in range(count):
+
+                def at(child: _Node, index: int = i) -> Any:
+                    val = child.get(row)  # type: ignore[attr-defined]
+                    if isinstance(val, _builtin_list):
+                        return val[index]
+                    return val
+
+                pack_nodes(buf, node.fields, row, seen, at)
+        elif isinstance(node, _Utf8):
+            value = take(node)
+            seen[node.field_id] = value
+            raw = _utf8_payload(str(node.field_id), value)
+            buf.append(len(raw) & 0xFF)
+            buf.append((len(raw) >> 8) & 0xFF)
+            buf.extend(raw)
+        elif isinstance(node, _List):
+            _pack_list_items(buf, node.element, take(node))
+        elif isinstance(node, _Dict):
+            _pack_dict_items(buf, node.element, take(node))
+        elif isinstance(node, _Group):
+            pack_nodes(buf, node.fields, row, seen, get_value)
+        else:
+            raise TypeError(f"unknown field node: {type(node)!r}")

packbin/_scheme.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import Any, Generic, TypeVar
+
+from packbin._errors import ShortPacket, TrailingBytes, TypeMismatch, UnpackResult
+from packbin._nodes import _Node, _validate_order
+from packbin._pack import pack_nodes
+from packbin._unpack import unpack_nodes
+
+T = TypeVar("T")
+_builtin_bytes = bytes
+_builtin_dict = dict
+
+
+class Scheme(Generic[T]):
+    __slots__ = ("_type_number", "_row_type", "_fields")
+
+    def __init__(self, type_number: int, row_type: type[T], *fields: _Node) -> None:
+        if isinstance(type_number, bool) or not isinstance(type_number, int) or type_number < 0 or type_number > 255:
+            raise ValueError(f"type number must be 0..255, got {type_number!r}")
+        _validate_order(fields)
+        self._type_number = type_number
+        self._row_type = row_type
+        self._fields = list(fields)
+
+    def on(self, handler: Callable[[T], None]) -> _Handler[T]:
+        return _Handler(self, handler)
+
+
+class _Handler(Generic[T]):
+    __slots__ = ("scheme", "handler")
+
+    def __init__(self, scheme: Scheme[T], handler: Callable[[T], None]) -> None:
+        self.scheme = scheme
+        self.handler = handler
+
+
+def _new_row(row_type: type[T]) -> T:
+    if row_type is _builtin_dict:
+        return {}  # type: ignore[return-value]
+    return row_type()
+
+
+class BinaryPacker:
+    @staticmethod
+    def pack(scheme: Scheme[T], row: T | Mapping[str, Any]) -> bytes:
+        buf = bytearray()
+        buf.append(scheme._type_number)
+        pack_nodes(buf, scheme._fields, row)
+        return _builtin_bytes(buf)
+
+    @staticmethod
+    def unpack(
+        first: Scheme[T] | bytes | bytearray | memoryview,
+        second: bytes | bytearray | memoryview | _Handler[Any] | None = None,
+        *rest: _Handler[Any],
+    ) -> UnpackResult[Any]:
+        if isinstance(first, Scheme):
+            if second is None:
+                raise TypeError("unpack() missing data")
+            if isinstance(second, _Handler):
+                raise TypeError("known unpack expects bytes")
+            return BinaryPacker._unpack_known(first, second)
+        if second is None and not rest:
+            raise TypeError("unpack() missing handlers")
+        handlers: list[_Handler[Any]] = []
+        if second is not None:
+            if not isinstance(second, _Handler):
+                raise TypeError("unknown unpack expects handlers")
+            handlers.append(second)
+        handlers.extend(rest)
+        return BinaryPacker._unpack_dispatch(first, tuple(handlers))
+
+    @staticmethod
+    def _unpack_fields(scheme: Scheme[T], view: memoryview, offset: int) -> UnpackResult[T]:
+        row = _new_row(scheme._row_type)
+        offset, err = unpack_nodes(view, offset, scheme._fields, row)
+        if err is not None:
+            return UnpackResult(ok=False, value=None, error=err)
+        left = len(view) - offset
+        if left > 0:
+            return UnpackResult(ok=False, value=None, error=TrailingBytes(left=left))
+        return UnpackResult(ok=True, value=row, error=None)
+
+    @staticmethod
+    def _unpack_known(scheme: Scheme[T], data: bytes | bytearray | memoryview) -> UnpackResult[T]:
+        view = memoryview(data)
+        left = len(view)
+        if left < 1:
+            return UnpackResult(ok=False, value=None, error=ShortPacket(field="", needed=1, left=left))
+        actual = int(view[0])
+        if actual != scheme._type_number:
+            return UnpackResult(ok=False, value=None, error=TypeMismatch(expected=scheme._type_number, actual=actual))
+        return BinaryPacker._unpack_fields(scheme, view, 1)
+
+    @staticmethod
+    def _unpack_dispatch(
+        data: bytes | bytearray | memoryview,
+        handlers: tuple[_Handler[Any], ...],
+    ) -> UnpackResult[Any]:
+        by_type: dict[int, _Handler[Any]] = {}
+        for handler in handlers:
+            number = handler.scheme._type_number
+            if number in by_type:
+                raise ValueError(f"duplicate type number {number}")
+            by_type[number] = handler
+        view = memoryview(data)
+        left = len(view)
+        if left < 1:
+            return UnpackResult(ok=False, value=None, error=ShortPacket(field="", needed=1, left=left))
+        actual = int(view[0])
+        matched = by_type.get(actual)
+        if matched is None:
+            return UnpackResult(ok=False, value=None, error=TypeMismatch(expected=-1, actual=actual))
+        result = BinaryPacker._unpack_fields(matched.scheme, view, 1)
+        if not result.ok or result.value is None:
+            return result
+        matched.handler(result.value)
+        return result

packbin/_unpack.py

@@ -0,0 +1,327 @@
+from __future__ import annotations
+
+import struct
+from collections.abc import Sequence
+from typing import Any
+
+from packbin._errors import ShortPacket, TypeMismatch
+from packbin._nodes import (
+    _Bits,
+    _Bool,
+    _Bytes,
+    _Dict,
+    _FlagBit,
+    _FlagByte,
+    _Flags,
+    _Group,
+    _List,
+    _Node,
+    _Repeat,
+    _Scalar,
+    _Sized,
+    _U2,
+    _Utf8,
+    _When,
+)
+
+_builtin_bytes = bytes
+_builtin_list = list
+_builtin_dict = dict
+
+
+def _read_u2(data: memoryview, offset: int, count: int, label: str) -> tuple[list[int], int] | ShortPacket:
+    nbytes = (count + 3) // 4
+    left = len(data) - offset
+    if left < nbytes:
+        return ShortPacket(field=label, needed=nbytes, left=left)
+    out: list[int] = []
+    for i in range(count):
+        out.append((data[offset + i // 4] >> ((i % 4) * 2)) & 3)
+    return out, offset + nbytes
+
+
+def _read_bits(data: memoryview, offset: int, label: str, count: int) -> tuple[list[int], int] | ShortPacket:
+    nbytes = (count + 7) // 8
+    left = len(data) - offset
+    if left < nbytes:
+        return ShortPacket(field=label, needed=nbytes, left=left)
+    out = [((data[offset + i // 8] >> (i % 8)) & 1) for i in range(count)]
+    return out, offset + nbytes
+
+
+def _read_scalar(
+    data: memoryview, offset: int, field: _Scalar
+) -> tuple[Any, int] | ShortPacket:
+    left = len(data) - offset
+    if left < field.size:
+        return ShortPacket(field=str(field.field_id), needed=field.size, left=left)
+    value = struct.unpack_from(field.endian_fmt(), data, offset)[0]
+    return value, offset + field.size
+
+
+def _is_leaf(node: _Node) -> bool:
+    return isinstance(node, (_Scalar, _Bytes, _Utf8, _Bool, _Sized, _Bits))
+
+
+def _append(row: Any, node: Any, value: Any, as_list: bool) -> None:
+    if as_list:
+        cur = node.get(row)
+        if cur is None:
+            node.set(row, [value])
+        elif isinstance(cur, _builtin_list):
+            cur.append(value)
+        else:
+            node.set(row, [cur, value])
+    else:
+        node.set(row, value)
+
+
+def _unpack_leaf(
+    data: memoryview, offset: int, node: _Node, seen: dict[int, Any]
+) -> tuple[Any, int, ShortPacket | TypeMismatch | None]:
+    if isinstance(node, _Scalar):
+        got = _read_scalar(data, offset, node)
+        if isinstance(got, ShortPacket):
+            return None, offset, got
+        value, offset = got
+        seen[node.field_id] = value
+        return value, offset, None
+    if isinstance(node, _Bytes):
+        left = len(data) - offset
+        if left < node.size:
+            return None, offset, ShortPacket(field=str(node.field_id), needed=node.size, left=left)
+        raw = _builtin_bytes(data[offset : offset + node.size])
+        offset += node.size
+        seen[node.field_id] = raw
+        return raw, offset, None
+    if isinstance(node, _Utf8):
+        left = len(data) - offset
+        if left < 2:
+            return None, offset, ShortPacket(field=str(node.field_id), needed=2, left=left)
+        count = struct.unpack_from("<H", data, offset)[0]
+        offset += 2
+        left = len(data) - offset
+        if left < count:
+            return None, offset, ShortPacket(field=str(node.field_id), needed=count, left=left)
+        raw = _builtin_bytes(data[offset : offset + count])
+        offset += count
+        value = raw.decode("utf-8")
+        seen[node.field_id] = value
+        return value, offset, None
+    if isinstance(node, _Bool):
+        return True, offset, None
+    raise TypeError(f"not a leaf: {type(node)!r}")
+
+
+def _unpack_element(
+    data: memoryview, offset: int, element: _Node
+) -> tuple[Any, int, ShortPacket | TypeMismatch | None]:
+    if isinstance(element, _List):
+        return _unpack_list_items(data, offset, element.element)
+    if isinstance(element, _Dict):
+        return _unpack_dict_items(data, offset, element.element)
+    if _is_leaf(element):
+        return _unpack_leaf(data, offset, element, {})
+    child_row: dict[str, Any] = {}
+    offset, err = unpack_nodes(data, offset, [element], child_row)
+    if err is not None:
+        return None, offset, err
+    return child_row, offset, None
+
+
+def _unpack_list_items(
+    data: memoryview, offset: int, element: _Node
+) -> tuple[list[Any], int, ShortPacket | TypeMismatch | None]:
+    left = len(data) - offset
+    if left < 2:
+        return [], offset, ShortPacket(field="", needed=2, left=left)
+    count = struct.unpack_from("<H", data, offset)[0]
+    offset += 2
+    items: list[Any] = []
+    for _ in range(count):
+        value, offset, err = _unpack_element(data, offset, element)
+        if err is not None:
+            return items, offset, err
+        items.append(value)
+    return items, offset, None
+
+
+def _unpack_dict_items(
+    data: memoryview, offset: int, element: _Node
+) -> tuple[dict[str, Any], int, ShortPacket | TypeMismatch | None]:
+    left = len(data) - offset
+    if left < 2:
+        return {}, offset, ShortPacket(field="", needed=2, left=left)
+    count = struct.unpack_from("<H", data, offset)[0]
+    offset += 2
+    mapping: dict[str, Any] = {}
+    for _ in range(count):
+        left = len(data) - offset
+        if left < 2:
+            return mapping, offset, ShortPacket(field="", needed=2, left=left)
+        key_len = struct.unpack_from("<H", data, offset)[0]
+        offset += 2
+        left = len(data) - offset
+        if left < key_len:
+            return mapping, offset, ShortPacket(field="", needed=key_len, left=left)
+        key = _builtin_bytes(data[offset : offset + key_len]).decode("utf-8")
+        offset += key_len
+        value, offset, err = _unpack_element(data, offset, element)
+        if err is not None:
+            return mapping, offset, err
+        if key in mapping:
+            return mapping, offset, ShortPacket(field="", needed=0, left=0)
+        mapping[key] = value
+    return mapping, offset, None
+
+
+def unpack_nodes(
+    data: memoryview,
+    offset: int,
+    nodes: Sequence[_Node],
+    row: Any,
+    seen: dict[int, Any] | None = None,
+    *,
+    as_list: bool = False,
+    flag_state: dict[int, int] | None = None,
+) -> tuple[int, ShortPacket | TypeMismatch | None]:
+    if seen is None:
+        seen = {}
+    if flag_state is None:
+        flag_state = {}
+
+    for node in nodes:
+        if isinstance(node, _Scalar):
+            got = _read_scalar(data, offset, node)
+            if isinstance(got, ShortPacket):
+                return offset, got
+            value, offset = got
+            seen[node.field_id] = value
+            _append(row, node, value, as_list)
+        elif isinstance(node, _Bytes):
+            left = len(data) - offset
+            if left < node.size:
+                return offset, ShortPacket(field=str(node.field_id), needed=node.size, left=left)
+            raw = _builtin_bytes(data[offset : offset + node.size])
+            offset += node.size
+            seen[node.field_id] = raw
+            _append(row, node, raw, as_list)
+        elif isinstance(node, _Bool):
+            pass
+        elif isinstance(node, _Flags):
+            left = len(data) - offset
+            if left < 1:
+                return offset, ShortPacket(field="", needed=1, left=left)
+            flag = data[offset]
+            offset += 1
+            for i, child in enumerate(node.fields):
+                if flag & (1 << i):
+                    if isinstance(child, _Bool):
+                        seen[child.field_id] = True
+                        _append(row, child, True, as_list)
+                    elif isinstance(child, _Group):
+                        offset, err = unpack_nodes(
+                            data, offset, child.fields, row, seen, as_list=as_list, flag_state=flag_state
+                        )
+                        if err is not None:
+                            return offset, err
+                    else:
+                        offset, err = unpack_nodes(
+                            data, offset, [child], row, seen, as_list=as_list, flag_state=flag_state
+                        )
+                        if err is not None:
+                            return offset, err
+        elif isinstance(node, _FlagByte):
+            left = len(data) - offset
+            if left < 1:
+                return offset, ShortPacket(field="", needed=1, left=left)
+            flag = data[offset]
+            offset += 1
+            flag_state[id(node)] = flag
+        elif isinstance(node, _FlagBit):
+            flag = flag_state.get(id(node.owner))
+            if flag is None:
+                raise RuntimeError("flag bit before flag byte")
+            if flag & (1 << node.index):
+                offset, err = unpack_nodes(
+                    data, offset, [node.field], row, seen, as_list=as_list, flag_state=flag_state
+                )
+                if err is not None:
+                    return offset, err
+        elif isinstance(node, _When):
+            if seen.get(node.condition.field_id) == node.condition.value:
+                offset, err = unpack_nodes(
+                    data, offset, node.fields, row, seen, as_list=as_list, flag_state=flag_state
+                )
+                if err is not None:
+                    return offset, err
+        elif isinstance(node, _Repeat):
+            while offset < len(data):
+                offset, err = unpack_nodes(
+                    data, offset, node.fields, row, seen, as_list=True, flag_state=flag_state
+                )
+                if err is not None:
+                    return offset, err
+        elif isinstance(node, _Sized):
+            count = seen.get(node.count)
+            if not isinstance(count, int):
+                raise RuntimeError(f"{node.field_id}: count {node.count} is missing")
+            left = len(data) - offset
+            if left < count:
+                return offset, ShortPacket(field=str(node.field_id), needed=count, left=left)
+            raw = _builtin_bytes(data[offset : offset + count])
+            offset += count
+            seen[node.field_id] = raw
+            _append(row, node, raw, as_list)
+        elif isinstance(node, _U2):
+            label = str(node.slots[0].field_id) if node.slots else "0"
+            got = _read_u2(data, offset, len(node.slots), label)
+            if isinstance(got, ShortPacket):
+                return offset, got
+            values_u2, offset = got
+            for slot, value in zip(node.slots, values_u2, strict=True):
+                seen[slot.field_id] = value
+                _append(row, slot, value, as_list)
+        elif isinstance(node, _Bits):
+            count = seen.get(node.count)
+            if not isinstance(count, int):
+                raise RuntimeError(f"{node.field_id}: count {node.count} is missing")
+            got_bits = _read_bits(data, offset, str(node.field_id), count)
+            if isinstance(got_bits, ShortPacket):
+                return offset, got_bits
+            bits_value, offset = got_bits
+            seen[node.field_id] = bits_value
+            _append(row, node, bits_value, as_list)
+        elif isinstance(node, _Utf8):
+            left = len(data) - offset
+            if left < 2:
+                return offset, ShortPacket(field=str(node.field_id), needed=2, left=left)
+            count = struct.unpack_from("<H", data, offset)[0]
+            offset += 2
+            left = len(data) - offset
+            if left < count:
+                return offset, ShortPacket(field=str(node.field_id), needed=count, left=left)
+            raw = _builtin_bytes(data[offset : offset + count])
+            offset += count
+            value = raw.decode("utf-8")
+            seen[node.field_id] = value
+            _append(row, node, value, as_list)
+        elif isinstance(node, _List):
+            items, offset, err = _unpack_list_items(data, offset, node.element)
+            if err is not None:
+                return offset, err
+            _append(row, node, items, as_list)
+        elif isinstance(node, _Dict):
+            mapping, offset, err = _unpack_dict_items(data, offset, node.element)
+            if err is not None:
+                return offset, err
+            _append(row, node, mapping, as_list)
+        elif isinstance(node, _Group):
+            offset, err = unpack_nodes(
+                data, offset, node.fields, row, seen, as_list=as_list, flag_state=flag_state
+            )
+            if err is not None:
+                return offset, err
+        else:
+            raise TypeError(f"unknown field node: {type(node)!r}")
+    return offset, None

packbin-0.1.9.dist-info/METADATA

@@ -0,0 +1,195 @@
+Metadata-Version: 2.4
+Name: packbin
+Version: 0.1.9
+Summary: Pack and unpack a caller-owned field list
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# packbin
+Binary packing and unpacking across languages, declarative mapping, and zero overhead in the binary data.
+
+Both sides keep the same field list. The bytes are only the values.
+
+Can be used for WebSocket, TCP, UDP, and other means of efficient communication
+
+## Example
+
+Python → binary → TypeScript
+
+### Python
+
+```python
+from packbin import BinaryPacker, Scheme, flags, i16, i32, u8, u16
+
+class Position:
+    def __init__(self):
+        self.sid = 1
+        self.lat = 500_000_000
+        self.lon = 300_000_000
+        self.profile = 1
+        self.heading = None
+        self.speed = None
+        self.altitude = None
+
+def bind(name):
+    return (lambda row: getattr(row, name), lambda row, value: setattr(row, name, value))
+
+target = Scheme(
+    0x40,
+    Position,
+    u16(0, *bind("sid")),
+    i32(1, *bind("lat")),
+    i32(2, *bind("lon")),
+    u8(3, *bind("profile")),
+    flags(
+        u16(4, *bind("heading")),
+        u8(5, *bind("speed")),
+        i16(6, *bind("altitude")),
+    ),
+)
+
+raw = BinaryPacker.pack(target, Position())
+```
+
+```
+40 01 00 00 65 cd 1d 00 a3 e1 11 01 00
+```
+
+### TypeScript
+
+```ts
+import { BinaryPacker, flags, i16, i32, scheme, u8, u16 } from "packbin"
+
+class Target {
+  sid = 1
+  lat = 500_000_000
+  lon = 300_000_000
+  profile = 1
+  heading: number | null = null
+  speed: number | null = null
+  altitude: number | null = null
+}
+
+const target = scheme<Target>(
+  0x40,
+  u16(0, (x) => x.sid),
+  i32(1, (x) => x.lat),
+  i32(2, (x) => x.lon),
+  u8(3, (x) => x.profile),
+  flags([
+    u16(4, (x) => x.heading),
+    u8(5, (x) => x.speed),
+    i16(6, (x) => x.altitude),
+  ]),
+)
+
+const got = BinaryPacker.unpack(target, raw)
+```
+
+`flags` is how an optional field takes no space when you have no value for it. `heading`, `speed`, and `altitude` are measurements, so `0` is still a value and has to be written. `None` means the field is not in the packet.
+
+`motion` is always one byte in front of those fields. Bit 0 is `heading`, bit 1 is `speed`, bit 2 is `altitude`. A set bit writes that field next. A clear bit skips it.
+
+| Field | Type | Bytes when present | Values |
+|---|---|---|---|
+| `heading` | `u16` | 2 | 0 … 65535 |
+| `speed` | `u8` | 1 | 0 … 255 |
+| `altitude` | `i16` | 2 | −32768 … 32767 |
+
+In this example all three are `None`, so `motion` is `00` and those 5 bytes are absent. The packet is 13 bytes: type number 1, `sid` 2, `lat` 4, `lon` 4, `profile` 1, `motion` 1.
+
+```
+40          type
+01 00       sid
+00 65 cd 1d lat
+00 a3 e1 11 lon
+01          profile
+00          motion
+```
+
+`heading = 90` sets bit 0, so `motion` is `01` and `5a 00` follows it. `heading = 0` sets the same bit and writes `00 00`. `speed = 10` sets bit 1 and writes one byte. The present fields are written in the order listed, and only those.
+
+## Example
+
+C# → binary → Rust
+
+### C#
+
+```csharp
+using Packbin;
+
+sealed class User
+{
+    public string Username { get; set; } = "";
+    public List<Role> Roles { get; set; } = [];
+    public Dictionary<string, ActionList> Access { get; set; } = [];
+}
+
+sealed class Role { public string RoleName { get; set; } = ""; }
+sealed class ActionName { public string Action { get; set; } = ""; }
+sealed class ActionList { public List<ActionName> Actions { get; set; } = []; }
+
+var userScheme = new Scheme<User>(1, f => [
+    f.Utf8(0, x => x.Username),
+    f.List(x => x.Roles, r => r.Utf8(0, role => role.RoleName)),
+    f.Dict(x => x.Access, e => e.List(a => a.Actions, n => n.Utf8(0, action => action.Action)))]);
+
+var raw = BinaryPacker.Pack(userScheme, new User
+{
+    Username = "ada",
+    Roles = [new Role { RoleName = "user" }, new Role { RoleName = "admin" }],
+    Access = new()
+    {
+        ["map"] = new ActionList { Actions = [new ActionName { Action = "read" }, new ActionName { Action = "edit" }] },
+        ["store"] = new ActionList { Actions = [new ActionName { Action = "write" }] },
+    },
+});
+```
+
+```
+01 03 00 61 64 61 02 00 04 00 75 73 65 72 05 00 61 64 6d 69 6e
+02 00 03 00 6d 61 70 02 00 04 00 72 65 61 64 04 00 65 64 69 74
+05 00 73 74 6f 72 65 01 00 05 00 77 72 69 74 65
+```
+
+The first byte is the scheme type number.
+
+### Rust
+
+```rust
+use packbin::{dict, list, unpack_map, utf8, MapScheme};
+
+let user = MapScheme::new(
+    1,
+    vec![
+        utf8("username"),
+        list("roles", utf8("role")),
+        dict("access", list("actions", utf8("action"))),
+    ],
+);
+
+let got = unpack_map(&user, &raw).unwrap();
+```
+
+## Data types
+
+| Helper | What it writes |
+|--------|----------------|
+| `u8` `u16` `u32` `u64` | unsigned integer, little-endian |
+| `i8` `i16` `i32` `i64` | signed integer, little-endian |
+| `f32` `f64` | IEEE 754 float |
+| `bytes(n)` | exactly `n` raw bytes |
+| `be(field)` | that number, big-endian |
+| `utf8` | UTF-8 string, `u16` length |
+| `list` | `u16` count, then that many elements |
+| `dict` | `u16` pair count; keys in unsigned byte order |
+| `flags(name, fields)` | one `u8`; bit 0 is the first field; a clear bit omits that field |
+| `when(eq(field, value), fields)` | the group only when an earlier field equals `value` |
+| `repeat(fields)` | the group until the buffer ends |
+
+[`_docs/01_solution/schema.md`](_docs/01_solution/schema.md)
+
+## License
+
+MIT

packbin-0.1.9.dist-info/RECORD

@@ -0,0 +1,10 @@
+packbin/__init__.py,sha256=sVzv7vjrU0CgXTv_BKy9nygjRnV3t2USSh1fJF1tMJE,864
+packbin/_errors.py,sha256=Gslv_uomJcsKsE_iCQneSf_-biZYgDZwbnbL-VxNKr8,1048
+packbin/_nodes.py,sha256=sje7mm0mpkd93ZKw1R3iAvnuUdvMvaHZQ0cU_7-ygSM,7783
+packbin/_pack.py,sha256=XeqiacohhxV6Ca3EZuffK0sQ-ZNgWoSXEZexE_l67wM,9536
+packbin/_scheme.py,sha256=-TuzNgCeFr4nOvpOELENbrAMmeUv18mHnDPpThlXOyg,4655
+packbin/_unpack.py,sha256=3kwQQhiBRLWmcoO6ji7RV9hbQ_gP5xOTXRq6eVS6w5A,12452
+packbin-0.1.9.dist-info/METADATA,sha256=hFKlTkx9_p4nl8P02ILQzKRWHCCs8UntgxUcT_yRjlA,5382
+packbin-0.1.9.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+packbin-0.1.9.dist-info/top_level.txt,sha256=dwrq4kGidAMiDQvoGgkmTpLcJe7ZvQdlxYHsLOPBiek,8
+packbin-0.1.9.dist-info/RECORD,,

packbin-0.1.9.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

packbin-0.1.9.dist-info/top_level.txt

@@ -0,0 +1 @@
+packbin