chimp-encoding 0.1.0__py3-none-any.whl

chimp_encoding/__init__.py

@@ -0,0 +1,17 @@
+"""Chimp time-series floating point compression for Python."""
+
+from chimp_encoding.chimp import ChimpDecoder, ChimpEncoder
+from chimp_encoding.chimp32 import Chimp32Decoder, Chimp32Encoder
+from chimp_encoding.chimp_n import ChimpNDecoder, ChimpNEncoder
+from chimp_encoding.chimp_n32 import ChimpN32Decoder, ChimpN32Encoder
+
+__all__ = [
+    "ChimpEncoder",
+    "ChimpDecoder",
+    "Chimp32Encoder",
+    "Chimp32Decoder",
+    "ChimpNEncoder",
+    "ChimpNDecoder",
+    "ChimpN32Encoder",
+    "ChimpN32Decoder",
+]

chimp_encoding/_bit_stream.py

@@ -0,0 +1,130 @@
+"""Big-endian bit-level I/O streams for Chimp compression."""
+
+from __future__ import annotations
+
+
+class OutputBitStream:
+    """Big-endian bit-level output stream over a growable byte buffer."""
+
+    def __init__(self, initial_capacity: int = 8000) -> None:
+        self._buffer: bytearray = bytearray(initial_capacity)
+        self._current: int = 0
+        self._free: int = 8
+        self._pos: int = 0
+
+    def _ensure_capacity(self, needed: int) -> None:
+        while self._pos + needed >= len(self._buffer):
+            self._buffer.extend(bytearray(len(self._buffer)))
+
+    def _write_in_current(self, value: int, length: int) -> None:
+        self._current |= (value & ((1 << length) - 1)) << (self._free - length)
+        self._free -= length
+        if self._free == 0:
+            self._ensure_capacity(1)
+            self._buffer[self._pos] = self._current & 0xFF
+            self._pos += 1
+            self._free = 8
+            self._current = 0
+
+    def write_bit(self, bit: bool) -> None:
+        self._write_in_current(1 if bit else 0, 1)
+
+    def write_int(self, value: int, length: int) -> None:
+        if length == 0:
+            return
+        if length <= self._free:
+            self._write_in_current(value, length)
+            return
+
+        # Fill current byte
+        remaining = length - self._free
+        if self._free != 0:
+            self._write_in_current(value >> remaining, self._free)
+
+        # Write full bytes
+        while remaining >= 8:
+            remaining -= 8
+            self._ensure_capacity(1)
+            self._buffer[self._pos] = (value >> remaining) & 0xFF
+            self._pos += 1
+
+        # Write remaining bits
+        if remaining > 0:
+            self._write_in_current(value, remaining)
+
+    def write_long(self, value: int, length: int) -> None:
+        self.write_int(value, length)
+
+    def flush(self) -> None:
+        if self._free < 8:
+            self._ensure_capacity(1)
+            self._buffer[self._pos] = self._current & 0xFF
+            self._pos += 1
+            self._current = 0
+            self._free = 8
+
+    @property
+    def buffer(self) -> bytes:
+        return bytes(self._buffer[: self._pos + (1 if self._free < 8 else 0)])
+
+
+class InputBitStream:
+    """Big-endian bit-level input stream over a byte buffer."""
+
+    def __init__(self, data: bytes | bytearray) -> None:
+        self._buffer: bytes = bytes(data)
+        self._current: int = 0
+        self._fill: int = 0
+        self._pos: int = 0
+
+    def _read_byte(self) -> int:
+        if self._pos >= len(self._buffer):
+            return 0
+        b = self._buffer[self._pos]
+        self._pos += 1
+        return b
+
+    def _refill(self) -> None:
+        while self._fill < 16 and self._pos < len(self._buffer):
+            self._current = (self._current << 8) | self._read_byte()
+            self._fill += 8
+
+    def _read_from_current(self, length: int) -> int:
+        if length == 0:
+            return 0
+        self._fill -= length
+        return (self._current >> self._fill) & ((1 << length) - 1)
+
+    def read_bit(self) -> int:
+        if self._fill == 0:
+            self._current = self._read_byte()
+            self._fill = 8
+        return self._read_from_current(1)
+
+    def read_int(self, length: int) -> int:
+        if length == 0:
+            return 0
+        if self._fill < 16:
+            self._refill()
+        if length <= self._fill:
+            return self._read_from_current(length)
+
+        remaining = length - self._fill
+        result = self._read_from_current(self._fill)
+
+        # Read full bytes
+        while remaining >= 8:
+            result = (result << 8) | self._read_byte()
+            remaining -= 8
+
+        # Read remaining bits
+        if remaining > 0:
+            if self._fill == 0:
+                self._current = self._read_byte()
+                self._fill = 8
+            result = (result << remaining) | self._read_from_current(remaining)
+
+        return result
+
+    def read_long(self, length: int) -> int:
+        return self.read_int(length)

chimp_encoding/_tables.py

@@ -0,0 +1,143 @@
+"""Shared lookup tables for Chimp encoding/decoding."""
+
+# Maps a leading-zero count (0..63) to a 3-bit representation code (0..7).
+# Used by encoders to quantize leading zero counts.
+LEADING_REPRESENTATION_ENCODE: tuple[int, ...] = (
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    1,
+    1,
+    1,
+    2,
+    2,
+    2,
+    2,
+    3,
+    3,
+    4,
+    4,
+    5,
+    5,
+    6,
+    6,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+    7,
+)
+
+# Rounds a leading-zero count to the nearest representable value.
+# Used by encoders before storing leading zeros.
+LEADING_ROUND: tuple[int, ...] = (
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    8,
+    8,
+    8,
+    8,
+    12,
+    12,
+    12,
+    12,
+    16,
+    16,
+    18,
+    18,
+    20,
+    20,
+    22,
+    22,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+    24,
+)
+
+# Inverse decode table: maps a 3-bit code back to the leading-zero count.
+# Used by decoders to reconstruct leading zero counts.
+LEADING_REPRESENTATION_DECODE: tuple[int, ...] = (0, 8, 12, 16, 18, 20, 22, 24)

chimp_encoding/chimp.py

@@ -0,0 +1,191 @@
+"""Chimp encoder/decoder for 64-bit doubles (base variant)."""
+
+from __future__ import annotations
+
+import struct
+
+from chimp_encoding._bit_stream import InputBitStream, OutputBitStream
+from chimp_encoding._tables import (
+    LEADING_REPRESENTATION_DECODE,
+    LEADING_REPRESENTATION_ENCODE,
+    LEADING_ROUND,
+)
+
+_NAN_LONG = 0x7FF8000000000000
+_MASK_64 = 0xFFFFFFFFFFFFFFFF
+
+
+def _double_to_long_bits(value: float) -> int:
+    return struct.unpack(">Q", struct.pack(">d", value))[0]
+
+
+def _long_bits_to_double(bits: int) -> float:
+    return struct.unpack(">d", struct.pack(">Q", bits & _MASK_64))[0]
+
+
+def _leading_zeros_64(x: int) -> int:
+    if x == 0:
+        return 64
+    return 64 - x.bit_length()
+
+
+def _trailing_zeros_64(x: int) -> int:
+    if x == 0:
+        return 64
+    return (x & -x).bit_length() - 1
+
+
+class ChimpEncoder:
+    """Compresses a series of 64-bit doubles using the Chimp algorithm."""
+
+    THRESHOLD: int = 6
+
+    def __init__(self) -> None:
+        self._out = OutputBitStream()
+        self._stored_leading_zeros: int = 65
+        self._stored_val: int = 0
+        self._first: bool = True
+        self._size: int = 0
+
+    def add_value(self, value: float) -> None:
+        bits = _double_to_long_bits(value)
+        if self._first:
+            self._write_first(bits)
+        else:
+            self._compress_value(bits)
+
+    def _write_first(self, value: int) -> None:
+        self._first = False
+        self._stored_val = value
+        self._out.write_long(value, 64)
+        self._size += 64
+
+    def _compress_value(self, value: int) -> None:
+        xor = (self._stored_val ^ value) & _MASK_64
+
+        if xor == 0:
+            self._out.write_bit(False)
+            self._out.write_bit(False)
+            self._size += 2
+            self._stored_leading_zeros = 65
+        else:
+            leading_zeros = LEADING_ROUND[_leading_zeros_64(xor)]
+            trailing_zeros = _trailing_zeros_64(xor)
+
+            if trailing_zeros > self.THRESHOLD:
+                significant_bits = 64 - leading_zeros - trailing_zeros
+                self._out.write_bit(False)
+                self._out.write_bit(True)
+                self._out.write_int(LEADING_REPRESENTATION_ENCODE[leading_zeros], 3)
+                self._out.write_int(significant_bits, 6)
+                self._out.write_long(xor >> trailing_zeros, significant_bits)
+                self._size += 11 + significant_bits
+                self._stored_leading_zeros = 65
+            elif leading_zeros == self._stored_leading_zeros:
+                self._out.write_bit(True)
+                self._out.write_bit(False)
+                significant_bits = 64 - leading_zeros
+                self._out.write_long(xor, significant_bits)
+                self._size += 2 + significant_bits
+            else:
+                self._stored_leading_zeros = leading_zeros
+                significant_bits = 64 - leading_zeros
+                self._out.write_bit(True)
+                self._out.write_bit(True)
+                self._out.write_int(LEADING_REPRESENTATION_ENCODE[leading_zeros], 3)
+                self._out.write_long(xor, significant_bits)
+                self._size += 5 + significant_bits
+
+        self._stored_val = value
+
+    def close(self) -> None:
+        self.add_value(float("nan"))
+        self._out.write_bit(False)
+        self._out.flush()
+
+    def get_size(self) -> int:
+        return self._size
+
+    def get_bytes(self) -> bytes:
+        return self._out.buffer
+
+
+class ChimpDecoder:
+    """Decompresses a Chimp-compressed stream of 64-bit doubles."""
+
+    def __init__(self, data: bytes) -> None:
+        self._in = InputBitStream(data)
+        self._stored_leading_zeros: int = 65
+        self._stored_trailing_zeros: int = 0
+        self._stored_val: int = 0
+        self._first: bool = True
+        self._end_of_stream: bool = False
+
+    def read_value(self) -> float | None:
+        if self._first:
+            self._first = False
+            self._stored_val = self._in.read_long(64)
+            if self._stored_val == _NAN_LONG:
+                self._end_of_stream = True
+                return None
+        else:
+            self._next_value()
+
+        if self._end_of_stream:
+            return None
+        return _long_bits_to_double(self._stored_val)
+
+    def _next_value(self) -> None:
+        flag = self._in.read_int(2)
+
+        if flag == 3:
+            # New leading zeros
+            self._stored_leading_zeros = LEADING_REPRESENTATION_DECODE[self._in.read_int(3)]
+            significant_bits = 64 - self._stored_leading_zeros
+            if significant_bits == 0:
+                significant_bits = 64
+            value = self._in.read_long(64 - self._stored_leading_zeros)
+            value = (self._stored_val ^ value) & _MASK_64
+            if value == _NAN_LONG:
+                self._end_of_stream = True
+            else:
+                self._stored_val = value
+
+        elif flag == 2:
+            # Same leading zeros
+            significant_bits = 64 - self._stored_leading_zeros
+            if significant_bits == 0:
+                significant_bits = 64
+            value = self._in.read_long(64 - self._stored_leading_zeros)
+            value = (self._stored_val ^ value) & _MASK_64
+            if value == _NAN_LONG:
+                self._end_of_stream = True
+            else:
+                self._stored_val = value
+
+        elif flag == 1:
+            # Trailing zeros > threshold
+            self._stored_leading_zeros = LEADING_REPRESENTATION_DECODE[self._in.read_int(3)]
+            significant_bits = self._in.read_int(6)
+            if significant_bits == 0:
+                significant_bits = 64
+            self._stored_trailing_zeros = 64 - significant_bits - self._stored_leading_zeros
+            value = self._in.read_long(
+                64 - self._stored_leading_zeros - self._stored_trailing_zeros
+            )
+            value = (value << self._stored_trailing_zeros) & _MASK_64
+            value = (self._stored_val ^ value) & _MASK_64
+            if value == _NAN_LONG:
+                self._end_of_stream = True
+            else:
+                self._stored_val = value
+
+        # flag == 0: identical value, nothing to do
+
+    def get_values(self) -> list[float]:
+        result: list[float] = []
+        value = self.read_value()
+        while value is not None:
+            result.append(value)
+            value = self.read_value()
+        return result

chimp_encoding/chimp32.py

@@ -0,0 +1,175 @@
+"""Chimp encoder/decoder for 32-bit floats (base variant)."""
+
+from __future__ import annotations
+
+import struct
+
+from chimp_encoding._bit_stream import InputBitStream, OutputBitStream
+from chimp_encoding._tables import (
+    LEADING_REPRESENTATION_DECODE,
+    LEADING_REPRESENTATION_ENCODE,
+    LEADING_ROUND,
+)
+
+_NAN_INT = 0x7FC00000
+_MASK_32 = 0xFFFFFFFF
+
+
+def _float_to_int_bits(value: float) -> int:
+    return struct.unpack(">I", struct.pack(">f", value))[0]
+
+
+def _int_bits_to_float(bits: int) -> float:
+    return struct.unpack(">f", struct.pack(">I", bits & _MASK_32))[0]
+
+
+def _leading_zeros_32(x: int) -> int:
+    if x == 0:
+        return 32
+    return 32 - x.bit_length()
+
+
+def _trailing_zeros_32(x: int) -> int:
+    if x == 0:
+        return 32
+    return (x & -x).bit_length() - 1
+
+
+class Chimp32Encoder:
+    """Compresses a series of 32-bit floats using the Chimp algorithm."""
+
+    THRESHOLD: int = 5
+
+    def __init__(self) -> None:
+        self._out = OutputBitStream()
+        self._stored_leading_zeros: int = 33
+        self._stored_val: int = 0
+        self._first: bool = True
+        self._size: int = 0
+
+    def add_value(self, value: float) -> None:
+        bits = _float_to_int_bits(value)
+        if self._first:
+            self._write_first(bits)
+        else:
+            self._compress_value(bits)
+
+    def _write_first(self, value: int) -> None:
+        self._first = False
+        self._stored_val = value
+        self._out.write_int(value, 32)
+        self._size += 32
+
+    def _compress_value(self, value: int) -> None:
+        xor = (self._stored_val ^ value) & _MASK_32
+
+        if xor == 0:
+            self._out.write_bit(False)
+            self._out.write_bit(False)
+            self._size += 2
+            self._stored_leading_zeros = 33
+        else:
+            leading_zeros = LEADING_ROUND[_leading_zeros_32(xor)]
+            trailing_zeros = _trailing_zeros_32(xor)
+
+            if trailing_zeros > self.THRESHOLD:
+                significant_bits = 32 - leading_zeros - trailing_zeros
+                self._out.write_bit(False)
+                self._out.write_bit(True)
+                self._out.write_int(LEADING_REPRESENTATION_ENCODE[leading_zeros], 3)
+                self._out.write_int(significant_bits, 5)
+                self._out.write_int(xor >> trailing_zeros, significant_bits)
+                self._size += 10 + significant_bits
+                self._stored_leading_zeros = 33
+            elif leading_zeros == self._stored_leading_zeros:
+                self._out.write_bit(True)
+                self._out.write_bit(False)
+                significant_bits = 32 - leading_zeros
+                self._out.write_int(xor, significant_bits)
+                self._size += 2 + significant_bits
+            else:
+                self._stored_leading_zeros = leading_zeros
+                significant_bits = 32 - leading_zeros
+                self._out.write_int(24 + LEADING_REPRESENTATION_ENCODE[leading_zeros], 5)
+                self._out.write_int(xor, significant_bits)
+                self._size += 5 + significant_bits
+
+        self._stored_val = value
+
+    def close(self) -> None:
+        self.add_value(float("nan"))
+        self._out.write_bit(False)
+        self._out.flush()
+
+    def get_size(self) -> int:
+        return self._size
+
+    def get_bytes(self) -> bytes:
+        return self._out.buffer
+
+
+class Chimp32Decoder:
+    """Decompresses a Chimp-compressed stream of 32-bit floats."""
+
+    def __init__(self, data: bytes) -> None:
+        self._in = InputBitStream(data)
+        self._stored_leading_zeros: int = 33
+        self._stored_trailing_zeros: int = 0
+        self._stored_val: int = 0
+        self._first: bool = True
+        self._end_of_stream: bool = False
+
+    def read_value(self) -> float | None:
+        if self._first:
+            self._first = False
+            self._stored_val = self._in.read_int(32)
+            if self._stored_val == _NAN_INT:
+                self._end_of_stream = True
+                return None
+        else:
+            self._next_value()
+
+        if self._end_of_stream:
+            return None
+        return _int_bits_to_float(self._stored_val)
+
+    def _next_value(self) -> None:
+        if self._in.read_bit() == 1:
+            if self._in.read_bit() == 1:
+                # New leading zeros
+                self._stored_leading_zeros = LEADING_REPRESENTATION_DECODE[self._in.read_int(3)]
+
+            significant_bits = 32 - self._stored_leading_zeros
+            if significant_bits == 0:
+                significant_bits = 32
+            value = self._in.read_int(32 - self._stored_leading_zeros)
+            value = (self._stored_val ^ value) & _MASK_32
+            if value == _NAN_INT:
+                self._end_of_stream = True
+            else:
+                self._stored_val = value
+
+        elif self._in.read_bit() == 1:
+            # Trailing zeros > threshold
+            self._stored_leading_zeros = LEADING_REPRESENTATION_DECODE[self._in.read_int(3)]
+            significant_bits = self._in.read_int(5)
+            if significant_bits == 0:
+                significant_bits = 32
+            self._stored_trailing_zeros = 32 - significant_bits - self._stored_leading_zeros
+            value = self._in.read_int(32 - self._stored_leading_zeros - self._stored_trailing_zeros)
+            value = (value << self._stored_trailing_zeros) & _MASK_32
+            value = (self._stored_val ^ value) & _MASK_32
+            if value == _NAN_INT:
+                self._end_of_stream = True
+            else:
+                self._stored_val = value
+
+        # else: identical value, nothing to do
+
+    def get_values(self) -> list[float]:
+        result: list[float] = []
+        value = self.read_value()
+        while value is not None:
+            result.append(value)
+            value = self.read_value()
+        return result