ltcodecs 4.3.0__tar.gz → 4.4.0__tar.gz

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/.gitignore

@@ -217,3 +217,5 @@ src/commandSender.py
 src/database/databases/hostCommands.db
 src/database/databases/remoteCommands.db
 src/uploadedFiles/.gitkeep
+
+.idea/

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/.gitlab-ci.yml

@@ -1,6 +1,7 @@
 include:
   - project: acomms/ci_templates
-    ref: v2.3.2
+    ref: v3.2.0
+    # Note: Don't forget to update CI_TEMPLATES_REF to match this
     file:
       - /templates/python-docs.yml
       - /templates/python-tests.yml
@@ -20,18 +21,20 @@ stages:
   - deploy
 
 variables:
+  # Must match the ci_templates include ref when using shared Dockerfiles.
+  CI_TEMPLATES_REF: "v3.2.0"
+
   PYTHON_DOCS_PDF_ARTIFACT: "ltcodecs.pdf"
   PYTHON_DOCS_PDF_SOURCE: "docs/_build/latex/ltcodecs.pdf"
 
   ROS_ACOMMS_REF: "devel"
 
-  PYTHON_PYTEST_ARGS: "-vv -rA --capture=tee-sys"
-  PYTHON_PYTEST_COVERAGE_ARGS: "--cov=ltcodecs"
-  PYTHON_PYTEST_TARGET: "tests/"
-
 native_tests:
   extends: .armw_native_pytest_template
   variables:
+    PYTHON_PYTEST_ARGS: "-vv -rA --capture=tee-sys"
+    PYTHON_PYTEST_COVERAGE_ARGS: "--cov=ltcodecs"
+    PYTHON_PYTEST_TARGET: "tests/"
     NATIVE_TEST_SETUP_COMMANDS: |
       pip install .
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ltcodecs
-Version: 4.3.0
+Version: 4.4.0
 Summary: LT Codecs: Lightweight codecs for use over low-throughput links
 Author: Woods Hole Oceanographic Institution, Acoustic Communications Group
 License: LGPL-3.0-or-later

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/codec_configs.rst

@@ -54,6 +54,19 @@ Common scalar aliases include ``string``, ``integer``, ``bool``, ``float``,
 natural constraint, and ``linspace_float`` when a physical step size or number
 of values is the natural constraint.
 
+Constants
+---------
+
+Use ``constant`` for a configured field value that is not sent on the wire.
+The codec emits zero bits, does not require the field to be present in the
+source dictionary, and returns its configured value during decoding.
+
+.. code-block:: yaml
+
+   protocol_version:
+     codec: constant
+     value: 2
+
 Nested Dictionaries
 -------------------
 
@@ -107,7 +120,7 @@ Variable-Length Arrays
 ----------------------
 
 Use ``variable_len_array`` for a list whose encoded length is included in the
-bit stream.
+bit stream by default.
 
 .. code-block:: yaml
 
@@ -117,7 +130,8 @@ bit stream.
      element_type: uint8
 
 ``max_length`` is required.  Values longer than ``max_length`` are currently
-truncated during encoding.
+truncated during encoding.  Set ``min_length`` when the list always has at
+least a known number of elements; encoding fails if a shorter list is provided.
 
 For element codecs that need parameters, put those parameters under
 ``element_params``:
@@ -142,6 +156,55 @@ Set ``nullable: true`` when an empty list should encode as a single bit.  When
      nullable: true
      element_type: uint8
 
+With ``nullable: true`` and the default ``length_mode: varint``, the initial
+bit indicates whether the list is longer than ``min_length``.  If it is not,
+the length prefix is omitted and the decoder reads exactly ``min_length``
+elements.
+
+Use ``length_mode: egint`` to encode the array length with exponential-Golomb
+coding instead of the fixed-width ``varint`` length codec:
+
+.. code-block:: yaml
+
+   sparse_samples:
+     codec: variable_len_array
+     min_length: 2
+     max_length: 10
+     length_mode: egint
+     element_type: uint8
+
+Use ``length_mode: fill_outer`` when an outer frame already determines the
+field size.  This omits the length prefix and decodes elements sequentially
+until the current bit stream is exhausted.  A ``fill_outer`` field must be the
+final encoded field in its containing ``dict`` or ``msg`` frame.
+
+.. code-block:: yaml
+
+   payload:
+     codec: variable_len_array
+     max_length: 10
+     length_mode: fill_outer
+     element_type: uint8
+
+Bit-Length Byte Arrays
+----------------------
+
+``variable_len_bit_array`` is a compact codec for a ``uint8`` payload with an
+exact bit length. It replaces a separately encoded size field with a three-bit
+unused-tail count.
+
+.. code-block:: yaml
+
+   payload:
+     codec: variable_len_bit_array
+     max_length: 65535
+     bit_length_field: payload_size_in_bits
+
+At encode time, the source message supplies both ``payload`` and
+``payload_size_in_bits``. The bit-length field must not appear as another
+configured field. Decoding returns the payload and derives
+``payload_size_in_bits`` from the byte count and trailing unused-bit count.
+
 Fixed-Length Arrays
 -------------------
 
@@ -180,6 +243,28 @@ with ``element_params``.
          z:
            codec: float32
 
+An all-constant element message can carry only the array length. Each decoded
+element is still a fresh ROS message instance:
+
+.. code-block:: yaml
+
+   links:
+     codec: variable_len_array
+     max_length: 7
+     element_type: msg
+     element_params:
+       ros_type: ros_acomms_net_msgs/Link
+       fields:
+         source:
+           codec: constant
+           value: 0
+         dest:
+           codec: constant
+           value: 0
+         modem:
+           codec: constant
+           value: ""
+
 Optional Fields
 ---------------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/reference/codec_aliases.rst

@@ -45,8 +45,12 @@ Alias                     Codec Class
 ``string_enum``           :py:class:`~ltcodecs.string_enum_codec.StringEnumCodec`
 ``lzma``                  :py:class:`~ltcodecs.lzma_codec.LzmaCodec`
 ``optional``              :py:class:`~ltcodecs.optional_field_codec.OptionalFieldCodec`
+``constant``              :py:class:`~ltcodecs.constant_codec.ConstantCodec`
 ========================  ====================================================
 
+``variable_len_bit_array``
+   :py:class:`~ltcodecs.variable_len_bit_array_codec.VariableLenBitArrayCodec`
+
 Metadata Aliases
 ----------------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/reference/codecs.rst

@@ -73,8 +73,17 @@ Container Codecs
 
 ``variable_len_array``
    Encodes a list with a length prefix.  ``max_length`` controls the largest
-   representable list.  ``element_type`` and ``element_params`` configure the
-   element codec.
+   representable list, and ``min_length`` can require a lower bound.  The
+   length prefix uses ``length_codec: varint`` by default; use
+   ``length_codec: egint`` when exponential-Golomb coding is a better fit for
+   the expected length distribution.  ``element_type`` and ``element_params``
+   configure the element codec.
+
+``variable_len_bit_array``
+   Encodes a variable-length ``uint8`` array and derives an exact bit length
+   for a second field. The wire format contains the array length, byte values,
+   and a three-bit count of unused final-byte bits. Configure the derived field
+   with ``bit_length_field``; it is not a separate configured wire field.
 
 ``fixed_len_array``
    Encodes a fixed number of elements.  The encoded bitstream does not include
@@ -84,6 +93,11 @@ Container Codecs
    Encodes a boolean presence field.  When true, the configured
    ``target_fields`` are encoded immediately after the presence bit.
 
+``constant``
+   Returns its configured ``value`` without consuming payload bits. This is
+   useful for decoded fields that are known by configuration rather than sent
+   on the wire. Constant fields do not need a source value during encoding.
+
 Special-Purpose Codecs
 ----------------------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/reference/config_schema.rst

@@ -37,6 +37,16 @@ Common Field Keys
    Maximum number of characters, bytes, or array elements for variable-length
    codecs.
 
+``min_length``
+   Minimum number of elements for ``variable_len_array``.  Encoding fails when
+   the provided list is shorter than this value.
+
+``length_mode``
+   Length handling mode for ``string``, ``bytes``, and
+   ``variable_len_array``.  Supported values are ``varint``, ``egint``, and
+   ``fill_outer``.  ``fill_outer`` omits the length prefix and relies on the
+   containing frame to define the field size.
+
 ``length``
    Number of elements for ``fixed_len_array``.
 
@@ -46,6 +56,14 @@ Common Field Keys
 ``element_params``
    Nested configuration passed to the element codec for an array.
 
+``bit_length_field``
+   Name of the derived bit-length field emitted by ``variable_len_bit_array``
+   during decoding. The named field is supplied when encoding but must not be
+   configured separately in the same mapping.
+
+``value``
+   YAML-native value returned by the zero-bit ``constant`` codec.
+
 ``nullable``
    When true on ``variable_len_array``, an empty list is encoded as a single
    absent/present bit.
@@ -91,15 +109,52 @@ Nested ROS messages use ``ros_type`` and may optionally provide ``fields``:
 Arrays
 ------
 
-Variable-length arrays include a length prefix:
+Variable-length arrays include a length prefix by default:
+
+.. code-block:: yaml
+
+   samples:
+     codec: variable_len_array
+     max_length: 8
+     element_type: uint16
+
+Use ``min_length`` when every encoded array has at least a known number of
+elements.  With ``nullable: true`` and ``length_mode: varint``, a false flag
+means the array has exactly ``min_length`` elements, so the length prefix is
+omitted in that case.
+
+.. code-block:: yaml
+
+   samples:
+     codec: variable_len_array
+     min_length: 2
+     max_length: 8
+     nullable: true
+     element_type: uint16
+
+Use ``length_mode: egint`` to encode lengths with exponential-Golomb coding:
 
 .. code-block:: yaml
 
    samples:
      codec: variable_len_array
+     min_length: 2
      max_length: 8
+     length_mode: egint
      element_type: uint16
 
+Use ``length_mode: fill_outer`` for strings, bytes, or variable-length arrays
+whose size is defined by an outer frame rather than an internal prefix.  A
+``fill_outer`` field must be the final encoded field in its containing
+``dict`` or ``msg`` frame.
+
+.. code-block:: yaml
+
+   payload:
+     codec: bytes
+     max_length: 32
+     length_mode: fill_outer
+
 Fixed-length arrays omit the length prefix:
 
 .. code-block:: yaml
@@ -138,6 +193,45 @@ Arrays of ROS messages use ``element_type: msg``:
          z:
            codec: float32
 
+Bit-Length Byte Arrays
+----------------------
+
+Use ``variable_len_bit_array`` for a ``uint8`` payload whose last byte can
+contain between zero and seven unused bits. The configuration key is the byte
+array field. ``bit_length_field`` names the related value that is derived on
+decode, so it is omitted as a separate field configuration.
+
+.. code-block:: yaml
+
+   payload:
+     codec: variable_len_bit_array
+     max_length: 65535
+     bit_length_field: payload_size_in_bits
+
+For a two-byte payload with ``payload_size_in_bits: 13``, the codec writes the
+normal length prefix and two bytes followed by a three-bit value of ``3``. On
+decode it returns the byte array and derives ``payload_size_in_bits`` as
+``13``. The supplied bit length must describe the final payload byte: it must
+be between ``8 * len(payload) - 7`` and ``8 * len(payload)``, or zero for an
+empty payload.
+
+Constants
+---------
+
+Use ``constant`` for a configured value that consumes no bits. The value is
+returned during decoding and replaces any source value during encoding.
+
+.. code-block:: yaml
+
+   source:
+     codec: constant
+     value: 0
+
+This is especially useful in explicit nested ROS message configurations. A
+``msg`` codec whose configured children are all constants can be encoded
+without an element object; a variable-length array of those messages can use
+``None`` placeholders solely to communicate its length.
+
 Top-Level Options
 -----------------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/reference/python_api.rst

@@ -20,6 +20,14 @@ ltcodecs.bytes\_codec module
    :undoc-members:
    :show-inheritance:
 
+ltcodecs.constant\_codec module
+-------------------------------
+
+.. automodule:: ltcodecs.constant_codec
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 ltcodecs.ccl\_latlon\_bcd\_codec module
 ---------------------------------------
 
@@ -189,6 +197,14 @@ ltcodecs.variable\_len\_array\_codec module
    :undoc-members:
    :show-inheritance:
 
+ltcodecs.variable\_len\_bit\_array\_codec module
+---------------------------------------------------
+
+.. automodule:: ltcodecs.variable_len_bit_array_codec
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 ltcodecs.varint\_codec module
 -----------------------------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/docs/ros_messages.rst

@@ -114,6 +114,18 @@ is treated like this explicit configuration:
 The default maximum length for inferred variable-length arrays is currently
 ``10``.  Use an explicit config when that limit should be different.
 
+For a ROS ``uint8`` payload whose exact bit length is also needed, use the
+explicit ``variable_len_bit_array`` codec. The payload field remains a normal
+ROS array; the named bit-length field is derived during decoding rather than
+encoded as a separate field.
+
+.. code-block:: yaml
+
+   payload:
+     codec: variable_len_bit_array
+     max_length: 65535
+     bit_length_field: payload_size_in_bits
+
 Nested Message Fields
 ---------------------
 
@@ -137,6 +149,32 @@ encoded or when individual nested fields need custom codecs.
 Without a ``fields`` mapping, ``RosMsgFieldCodec`` infers the nested message's
 fields from the ROS message definition.
 
+Constant Nested Fields
+----------------------
+
+Use ``constant`` to reconstruct nested fields without transmitting them. A
+constant field consumes zero bits and returns its configured YAML value on
+decode.
+
+.. code-block:: yaml
+
+   link:
+     codec: msg
+     ros_type: ros_acomms_net_msgs/Link
+     fields:
+       source:
+         codec: constant
+         value: 0
+       dest:
+         codec: constant
+         value: 0
+
+When every configured child of a nested ``msg`` is constant, the message codec
+does not require a source message while encoding. This permits a
+``variable_len_array`` of such messages to use ``None`` values as input solely
+to represent the desired array length; decoding constructs a fresh ROS message
+for every element.
+
 Metadata
 --------
 

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/pyproject.toml

@@ -26,7 +26,7 @@ homepage = "https://git.whoi.edu/acomms/ltcodec/"
 documentation = "https://acomms.pages.whoi.edu/ltcodec/"
 
 [project.scripts]
-# No scripts for now; we may add scripts in /bin in a future release
+inspect_codec = "ltcodecs.inspect_cli:main"
 
 [project.optional-dependencies]
 # No optional dependencies
@@ -36,4 +36,4 @@ requires = ["setuptools>=64", "wheel", "setuptools_scm>=8"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
-write_to = "_version.py"
+write_to = "_version.py"

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/src/ltcodecs/__init__.py

@@ -10,19 +10,23 @@ from .bool_codec import BoolCodec
 from .string_codecs import AsciiStringCodec
 from .string_enum_codec import StringEnumCodec
 from .bytes_codec import BytesCodec
+from .metadata_field_codec import MetadataFieldCodec
 from .variable_len_array_codec import VariableLenArrayCodec
+from .variable_len_bit_array_codec import VariableLenBitArrayCodec
 from .fixed_len_array_codec import FixedLenArrayCodec
 from .dict_field_codec import DictFieldCodec
 from .ros_msg_field_codec import RosMsgFieldCodec
 from .ccl_latlon_codec import CclLatLonCodec
 from .ccl_latlon_bcd_codec import CclLatLonBcdCodec
 from .padding_codec import PaddingCodec
+from .constant_codec import ConstantCodec
 from .rostime_codec import RosTimeCodec
 from .dict_codec import DictCodec
 from .ros_message_codec import RosMessageCodec
 from .lzma_codec import LzmaCodec
 from .optional_field_codec import OptionalFieldCodec
 from .exceptions import EncodingFailed
+from .inspection import CodecInspector, FieldReport, InspectionReport, ValidationIssue
 
 field_codec_classes = {
     "integer": VarintCodec,
@@ -52,11 +56,13 @@ field_codec_classes = {
     "msg": RosMsgFieldCodec,
     "ros_msg": RosMsgFieldCodec,
     "variable_len_array": VariableLenArrayCodec,
+    "variable_len_bit_array": VariableLenBitArrayCodec,
     "fixed_len_array": FixedLenArrayCodec,
     "ccl_latlon": CclLatLonCodec,
     "ccl_latlon_bcd": CclLatLonBcdCodec,
     "pad": PaddingCodec,
     "padding": PaddingCodec,
+    "constant": ConstantCodec,
     "time": RosTimeCodec,
     "rostime": RosTimeCodec,
     "string_enum": StringEnumCodec,

ltcodecs-4.4.0/src/ltcodecs/_length_modes.py

@@ -0,0 +1,61 @@
+"""
+Shared helpers for codecs that encode variable-length values.
+"""
+
+from __future__ import annotations
+
+from bitstring import BitArray, ConstBitStream
+
+from .exponentialgolomb_integer_codec import ExponentialGolombIntegerCodec
+from .varint_codec import VarintCodec
+
+
+VALID_LENGTH_MODES = ("varint", "egint", "fill_outer")
+PREFIXED_LENGTH_MODES = ("varint", "egint")
+FILL_OUTER_LENGTH_MODE = "fill_outer"
+
+
+def validate_length_mode(length_mode: str, codec_name: str) -> str:
+    if length_mode not in VALID_LENGTH_MODES:
+        raise ValueError(
+            f"{codec_name} has invalid length_mode parameter: {length_mode}, "
+            f"should be one of {', '.join(VALID_LENGTH_MODES)}."
+        )
+    return length_mode
+
+
+def make_length_codec(length_mode: str, min_value: int, max_value: int):
+    if length_mode == "varint":
+        return VarintCodec(min_value=min_value, max_value=max_value)
+    if length_mode == "egint":
+        return ExponentialGolombIntegerCodec(min_value=min_value, max_value=max_value)
+    return None
+
+
+def encode_length(length_mode: str, length_codec, length: int) -> BitArray:
+    if length_mode == FILL_OUTER_LENGTH_MODE:
+        return BitArray()
+    length_bits, _ = length_codec.encode(length)
+    return BitArray(length_bits)
+
+
+def decode_length(length_mode: str, length_codec, bits_to_decode: ConstBitStream) -> int:
+    if length_mode == FILL_OUTER_LENGTH_MODE:
+        raise ValueError("fill_outer length mode does not encode a length prefix")
+    return length_codec.decode(bits_to_decode)
+
+
+def remaining_bits(bits_to_decode: ConstBitStream) -> int:
+    return len(bits_to_decode) - bits_to_decode.pos
+
+
+def prefix_min_length_bits(length_mode: str, length_codec) -> int:
+    if length_mode == FILL_OUTER_LENGTH_MODE:
+        return 0
+    return length_codec.min_length_bits
+
+
+def prefix_max_length_bits(length_mode: str, length_codec) -> int:
+    if length_mode == FILL_OUTER_LENGTH_MODE:
+        return 0
+    return length_codec.max_length_bits

{ltcodecs-4.3.0 → ltcodecs-4.4.0}/src/ltcodecs/bytes_codec.py

@@ -8,7 +8,16 @@ This module contains the BytesCodec class, which is used to encode and decode by
 from __future__ import annotations
 from bitstring import BitArray, ConstBitStream, Bits
 from .field_codec import FieldCodec
-from .varint_codec import VarintCodec
+from ._length_modes import (
+    FILL_OUTER_LENGTH_MODE,
+    decode_length,
+    encode_length,
+    make_length_codec,
+    prefix_max_length_bits,
+    prefix_min_length_bits,
+    remaining_bits,
+    validate_length_mode,
+)
 from .exceptions import EncodingFailed
 
 
@@ -17,9 +26,10 @@ class BytesCodec(FieldCodec):
     codec for bytes
     """
 
-    def __init__(self, max_length: int, fail_on_overflow=False, **kwargs) -> None:
+    def __init__(self, max_length: int, fail_on_overflow=False, length_mode: str = "varint", **kwargs) -> None:
         self.max_length = max_length
-        self.length_codec = VarintCodec(min_value=0, max_value=self.max_length)
+        self.length_mode = validate_length_mode(length_mode, "BytesCodec")
+        self.length_codec = make_length_codec(self.length_mode, min_value=0, max_value=self.max_length)
         self.fail_on_overflow = fail_on_overflow
 
     def encode(self, value: bytes) -> tuple[Bits, bytes]:
@@ -33,9 +43,8 @@ class BytesCodec(FieldCodec):
                 f"BytesCodec: value with length {len(value)} is too long to encode (codec max_length={self.max_length}"
             )
         value = value[0 : self.max_length]
-        length_bits, _ = self.length_codec.encode(len(value))
         value_bits = BitArray(bytes=value)
-        value_bits.prepend(length_bits)
+        value_bits.prepend(encode_length(self.length_mode, self.length_codec, len(value)))
         return value_bits, value
 
     def decode(self, bits_to_decode: ConstBitStream) -> bytes:
@@ -44,14 +53,24 @@ class BytesCodec(FieldCodec):
 
         :param bits_to_decode: the bits to decode
         """
-        num_bytes = self.length_codec.decode(bits_to_decode)
+        if self.length_mode == FILL_OUTER_LENGTH_MODE:
+            num_bits = remaining_bits(bits_to_decode)
+            if num_bits % 8:
+                raise ValueError(
+                    "BytesCodec length_mode='fill_outer' received bits that do not align to bytes."
+                )
+            num_bytes = num_bits // 8
+            if num_bytes > self.max_length:
+                raise ValueError(f"BytesCodec decoded {num_bytes} bytes, exceeding max_length={self.max_length}.")
+        else:
+            num_bytes = decode_length(self.length_mode, self.length_codec, bits_to_decode)
         value = bits_to_decode.read("bytes:" + str(num_bytes))
         return value
 
     @property
     def min_length_bits(self) -> int:
-        return self.length_codec.max_length_bits
+        return prefix_min_length_bits(self.length_mode, self.length_codec)
 
     @property
     def max_length_bits(self) -> int:
-        return self.length_codec.max_length_bits + (8 * self.max_length)
+        return prefix_max_length_bits(self.length_mode, self.length_codec) + (8 * self.max_length)

ltcodecs-4.4.0/src/ltcodecs/constant_codec.py

@@ -0,0 +1,41 @@
+"""Zero-bit codec for configured constant values."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any
+
+from bitstring import BitArray, ConstBitStream
+
+from .field_codec import FieldCodec
+
+
+class ConstantCodec(FieldCodec):
+    """Return a configured value without encoding it on the wire.
+
+    The source message need not contain a field using this codec. Each encode
+    and decode operation returns a deep copy so mutable configured values are
+    never shared between decoded messages.
+
+    Args:
+        value: YAML-native value returned during decoding.
+    """
+
+    requires_encode_value = False
+
+    def __init__(self, value: Any, **kwargs: Any) -> None:
+        self.value = value
+
+    def encode(self, value: Any = None) -> tuple[BitArray, Any]:
+        return BitArray(), deepcopy(self.value)
+
+    def decode(self, bits_to_decode: ConstBitStream) -> Any:
+        return deepcopy(self.value)
+
+    @property
+    def min_length_bits(self) -> int:
+        return 0
+
+    @property
+    def max_length_bits(self) -> int:
+        return 0