pureini 0.1.1__tar.gz

pureini-0.1.1/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.3
+Name: pureini
+Version: 0.1.1
+Summary: Add your description here
+Author: Marko Bausch
+Author-email: Marko Bausch <60338487+mrkbac@users.noreply.github.com>
+Requires-Dist: lz4>=4.4.4
+Requires-Dist: numba>=0.62.1
+Requires-Dist: numpy>=2.2.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: zstandard>=0.25.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+

pureini-0.1.1/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "pureini"
+version = "0.1.1"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "Marko Bausch", email = "60338487+mrkbac@users.noreply.github.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "lz4>=4.4.4",
+    "numba>=0.62.1",
+    "numpy>=2.2.0",
+    "pyyaml>=6.0.3",
+    "zstandard>=0.25.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.9,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]

pureini-0.1.1/src/pureini/__init__.py

@@ -0,0 +1,32 @@
+"""
+Pureini - Pure Python implementation of Cloudini point cloud compression.
+
+Copyright 2025 Davide Faconti
+Licensed under the Apache License, Version 2.0
+"""
+
+from .decoder import PointcloudDecoder
+from .encoder import PointcloudEncoder
+from .header import HeaderEncoding, decode_header, encode_header
+from .types import (
+    CompressionOption,
+    EncodingInfo,
+    EncodingOptions,
+    FieldType,
+    PointField,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompressionOption",
+    "EncodingInfo",
+    "EncodingOptions",
+    "FieldType",
+    "HeaderEncoding",
+    "PointField",
+    "PointcloudDecoder",
+    "PointcloudEncoder",
+    "decode_header",
+    "encode_header",
+]

pureini-0.1.1/src/pureini/decoder.py

@@ -0,0 +1,156 @@
+"""
+Point cloud decoder for Pureini compression.
+
+Copyright 2025 Davide Faconti
+Licensed under the Apache License, Version 2.0
+"""
+
+import lz4.block
+import numpy as np
+import zstandard as zstd
+
+from .encoding_utils import BufferView, build_field_metadata, decode
+from .header import decode_header
+from .jit_codec import decode_chunk_jit
+from .types import (
+    POINTS_PER_CHUNK,
+    CompressionOption,
+    EncodingInfo,
+    EncodingOptions,
+)
+
+
+class PointcloudDecoder:
+    """
+    Point cloud decoder for two-stage decompression.
+
+    Stage 1: General-purpose decompression (LZ4 or ZSTD)
+    Stage 2: Field-specific decoding (delta, dequantization, etc.) via Numba JIT
+    """
+
+    def __init__(self) -> None:
+        """Initialize the decoder."""
+
+    def decode(self, compressed_data: bytes) -> tuple[bytes, EncodingInfo]:
+        """
+        Decode compressed point cloud data.
+
+        Args:
+            compressed_data: Compressed data including header
+
+        Returns:
+            Tuple of (decompressed_data, encoding_info)
+        """
+        # Decode header and get bytes consumed
+        info, header_size = decode_header(bytes(compressed_data))
+
+        # Create view starting after header
+        input_view = BufferView(compressed_data)
+        input_view.trim_front(header_size)
+
+        # Build field metadata arrays for JIT
+        field_offsets, field_types, field_resolutions = build_field_metadata(info)
+
+        # Allocate output buffer
+        total_points = info.width * info.height
+        output_size = total_points * info.point_step
+        output = np.zeros(output_size, dtype=np.uint8)
+        output_offset = 0
+
+        # Decode chunks (version 3+)
+        if info.version >= 3:
+            while not input_view.empty():
+                # Read chunk size
+                chunk_size = int(decode(input_view, "I"))  # uint32
+
+                if chunk_size > input_view.size():
+                    raise RuntimeError("Invalid chunk size found while decoding")
+
+                # Extract chunk
+                chunk_data = input_view.read_bytes(chunk_size)
+
+                # Decompress chunk
+                decompressed = self._decompress_chunk(info, chunk_data)
+
+                # Fast path for NONE encoding: copy raw bytes without transformation
+                if info.encoding_opt == EncodingOptions.NONE:
+                    # Copy raw bytes directly to output
+                    chunk_bytes = len(decompressed)
+                    points_decoded = chunk_bytes // info.point_step
+                    output[output_offset : output_offset + chunk_bytes] = np.frombuffer(
+                        decompressed, dtype=np.uint8
+                    )
+                    output_offset += chunk_bytes
+                else:
+                    # Decode chunk using JIT
+                    # Calculate max points we can decode into remaining output space
+                    remaining_output_bytes = output_size - output_offset
+                    max_points_to_decode = remaining_output_bytes // info.point_step
+
+                    output_chunk = output[output_offset:]
+                    points_decoded = decode_chunk_jit(
+                        memoryview(decompressed),
+                        max_points_to_decode,
+                        info.point_step,
+                        field_offsets,
+                        field_types,
+                        field_resolutions,
+                        output_chunk,
+                    )
+
+                    output_offset += points_decoded * info.point_step
+        else:
+            # Version 2: single chunk
+            decompressed = self._decompress_chunk(info, bytes(input_view.data))
+
+            # Fast path for NONE encoding: copy raw bytes without transformation
+            if info.encoding_opt == EncodingOptions.NONE:
+                chunk_bytes = len(decompressed)
+                output[:chunk_bytes] = np.frombuffer(decompressed, dtype=np.uint8)
+                output_offset = chunk_bytes
+            else:
+                max_points_to_decode = total_points
+                points_decoded = decode_chunk_jit(
+                    memoryview(decompressed),
+                    max_points_to_decode,
+                    info.point_step,
+                    field_offsets,
+                    field_types,
+                    field_resolutions,
+                    output,
+                )
+                output_offset = points_decoded * info.point_step
+
+        return bytes(output[:output_offset]), info
+
+    def _decompress_chunk(self, info: EncodingInfo, chunk_data: bytes) -> bytes:
+        """
+        Decompress a single chunk.
+
+        Args:
+            info: Encoding information
+            chunk_data: Compressed chunk data
+
+        Returns:
+            Decompressed chunk data
+        """
+        if info.compression_opt == CompressionOption.LZ4:
+            try:
+                max_size = POINTS_PER_CHUNK * info.point_step
+                return lz4.block.decompress(chunk_data, uncompressed_size=max_size)
+            except (ValueError, RuntimeError) as e:
+                raise RuntimeError(f"LZ4 decompression failed: {e}") from e
+
+        elif info.compression_opt == CompressionOption.ZSTD:
+            try:
+                dctx = zstd.ZstdDecompressor()
+                return dctx.decompress(chunk_data)
+            except (ValueError, RuntimeError, zstd.ZstdError) as e:
+                raise RuntimeError(f"ZSTD decompression failed: {e}") from e
+
+        elif info.compression_opt == CompressionOption.NONE:
+            # No decompression needed
+            return chunk_data
+
+        else:
+            raise RuntimeError(f"Unknown compression option: {info.compression_opt}")

pureini-0.1.1/src/pureini/encoder.py

@@ -0,0 +1,171 @@
+"""
+Point cloud encoder for Pureini compression.
+
+Copyright 2025 Davide Faconti
+Licensed under the Apache License, Version 2.0
+"""
+
+import struct
+
+import lz4.block
+import numpy as np
+import zstandard as zstd
+
+from .encoding_utils import BufferView, build_field_metadata
+from .header import HeaderEncoding, encode_header
+from .jit_codec import encode_chunk_jit
+from .types import POINTS_PER_CHUNK, CompressionOption, EncodingInfo, EncodingOptions
+
+
+def _zstd_compress_bound(src_size: int) -> int:
+    """
+    Calculate maximum compressed size for zstd compression.
+
+    Implements the ZSTD_compressBound formula from the C API:
+    srcSize + (srcSize >> 8) + margin_for_small_inputs
+
+    The Python zstandard library doesn't expose ZSTD_compressBound,
+    so we implement it based on the official zstd specification.
+
+    Args:
+        src_size: Size of uncompressed data in bytes
+
+    Returns:
+        Maximum size needed for compressed output buffer
+    """
+    base = src_size + (src_size >> 8)
+    # Add extra margin for inputs smaller than 128 KB
+    if src_size < (128 << 10):  # 128 KB = 131072 bytes
+        margin = ((128 << 10) - src_size) >> 11
+        return base + margin
+    return base
+
+
+class PointcloudEncoder:
+    """
+    Point cloud encoder using two-stage compression.
+
+    Stage 1: Field-specific encoding (delta, quantization, etc.) via Numba JIT
+    Stage 2: General-purpose compression (LZ4 or ZSTD)
+    """
+
+    def __init__(self, info: EncodingInfo) -> None:
+        """
+        Initialize the encoder with encoding configuration.
+
+        Args:
+            info: Encoding configuration
+        """
+        self.info = info
+        self.header = encode_header(info, HeaderEncoding.YAML)
+
+        # Build field metadata arrays for JIT
+        self.field_offsets, self.field_types, self.field_resolutions = build_field_metadata(info)
+
+        # Reuse compressor across chunks
+        if info.compression_opt == CompressionOption.ZSTD:
+            self._zstd_cctx = zstd.ZstdCompressor(level=1)
+
+    def encode(self, cloud_data: bytes) -> bytes:
+        """
+        Encode point cloud data.
+
+        Args:
+            cloud_data: Raw point cloud bytes
+
+        Returns:
+            Compressed point cloud data with header
+        """
+        # Convert to numpy array for JIT
+        point_data = np.frombuffer(cloud_data, dtype=np.uint8)
+
+        # Calculate points and chunks
+        points_count = len(cloud_data) // self.info.point_step
+        chunks_count = (points_count + POINTS_PER_CHUNK - 1) // POINTS_PER_CHUNK
+
+        # Worst-case compression bound
+        if self.info.compression_opt == CompressionOption.ZSTD:
+            max_compressed = _zstd_compress_bound(len(cloud_data))
+        else:
+            max_compressed = len(cloud_data) * 2  # Conservative estimate for LZ4
+
+        # Allocate output buffer: header + compressed data + chunk headers (4 bytes each)
+        output_size = len(self.header) + max_compressed + (4 * chunks_count)
+        output = bytearray(output_size)
+        output_view = BufferView(output)
+
+        # Write header
+        output_view.write_bytes(self.header)
+
+        # Encode and compress chunks
+        self._encode_chunks(point_data, points_count, output_view)
+
+        # Trim to actual size
+        actual_size = output_size - output_view.size()
+        return bytes(output[:actual_size])
+
+    def _encode_chunks(
+        self, point_data: np.ndarray, points_count: int, output_view: BufferView
+    ) -> None:
+        """
+        Encode point cloud data in chunks using JIT.
+
+        Args:
+            point_data: Raw point cloud data as numpy array
+            points_count: Total number of points
+            output_view: Output buffer for compressed chunks
+        """
+        # Allocate temporary buffer for encoded chunk (before compression)
+        chunk_buffer = np.zeros(POINTS_PER_CHUNK * self.info.point_step, dtype=np.uint8)
+
+        chunk_start = 0
+        while chunk_start < points_count:
+            # Determine chunk size
+            chunk_points = min(POINTS_PER_CHUNK, points_count - chunk_start)
+
+            # Fast path for NONE encoding: copy raw bytes without transformation
+            if self.info.encoding_opt == EncodingOptions.NONE:
+                # Calculate byte offsets
+                byte_start = chunk_start * self.info.point_step
+                byte_end = byte_start + (chunk_points * self.info.point_step)
+                chunk_data = bytes(point_data[byte_start:byte_end])
+            else:
+                # Call JIT encoder for this chunk
+                bytes_written = encode_chunk_jit(
+                    point_data,
+                    chunk_start,
+                    chunk_points,
+                    self.info.point_step,
+                    self.field_offsets,
+                    self.field_types,
+                    self.field_resolutions,
+                    chunk_buffer,
+                )
+                chunk_data = bytes(chunk_buffer[:bytes_written])
+
+            # Compress and write chunk
+            self._write_chunk(chunk_data, output_view)
+
+            chunk_start += chunk_points
+
+    def _write_chunk(self, chunk_data: bytes, output_view: BufferView) -> None:
+        """
+        Compress and write a chunk to output.
+
+        Args:
+            chunk_data: Encoded chunk data
+            output_view: Output buffer
+        """
+        if self.info.compression_opt == CompressionOption.LZ4:
+            payload = lz4.block.compress(chunk_data, store_size=False)
+        elif self.info.compression_opt == CompressionOption.ZSTD:
+            payload = self._zstd_cctx.compress(chunk_data)
+        elif self.info.compression_opt == CompressionOption.NONE:
+            payload = chunk_data
+        else:
+            raise RuntimeError(f"Unknown compression option: {self.info.compression_opt}")
+
+        # Write uint32 size + payload
+        struct.pack_into("<I", output_view.data, 0, len(payload))
+        output_view.trim_front(4)
+        output_view.write_bytes(payload)

pureini-0.1.1/src/pureini/encoding_utils.py

@@ -0,0 +1,288 @@
+"""
+Encoding utilities for Pureini point cloud compression.
+
+Copyright 2025 Davide Faconti
+Licensed under the Apache License, Version 2.0
+"""
+
+import struct
+
+import numba as nb
+import numpy as np
+
+from .types import EncodingInfo, EncodingOptions, FieldType
+
+
+class BufferView:
+    """
+    A view into a byte buffer that can be consumed as data is written/read.
+    Accepts bytes, bytearray, or memoryview. Write operations require a
+    writable underlying buffer (bytearray or writable memoryview).
+    """
+
+    def __init__(self, data: bytes | bytearray | memoryview) -> None:
+        if isinstance(data, (bytes, bytearray)):
+            self._view = memoryview(data)
+        else:
+            self._view = data
+        self._offset = 0
+
+    @property
+    def data(self) -> memoryview:
+        """Get the current view of remaining data."""
+        return self._view[self._offset :]
+
+    def size(self) -> int:
+        """Get the size of remaining data."""
+        return len(self._view) - self._offset
+
+    def empty(self) -> bool:
+        """Check if buffer is empty."""
+        return self.size() == 0
+
+    def trim_front(self, n: int) -> None:
+        if n > self.size():
+            raise RuntimeError(f"Cannot trim {n} bytes, only {self.size()} available")
+        self._offset += n
+
+    def write_bytes(self, data: bytes | bytearray) -> None:
+        n = len(data)
+        if n > self.size():
+            raise RuntimeError(f"Cannot write {n} bytes, only {self.size()} available")
+        self.data[:n] = data
+        self.trim_front(n)
+
+    def read_bytes(self, n: int) -> bytes:
+        if n > self.size():
+            raise RuntimeError(f"Cannot read {n} bytes, only {self.size()} available")
+        result = bytes(self.data[:n])
+        self.trim_front(n)
+        return result
+
+
+@nb.njit(cache=True, fastmath=True)
+def encode_varint64_to_buffer(value: int, buffer: memoryview, offset: int = 0) -> int:
+    """
+    Encode a signed 64-bit integer directly to a buffer (zero-copy).
+    Value 0 is reserved for NaN, so all values are shifted by +1.
+
+    JIT-compiled with Numba for maximum performance.
+
+    Args:
+        value: The signed 64-bit integer to encode
+        buffer: The target buffer (memoryview)
+        offset: Starting offset in the buffer
+
+    Returns:
+        Number of bytes written
+    """
+    # Zigzag encoding
+    val = value << 1 if value >= 0 else ((-value - 1) << 1) | 1
+
+    # Reserve value 0 for NaN
+    val += 1
+
+    # Varint encoding - write directly to buffer
+    ptr = offset
+    while val > 0x7F:
+        buffer[ptr] = (val & 0x7F) | 0x80
+        val >>= 7
+        ptr += 1
+    buffer[ptr] = val & 0xFF
+    ptr += 1
+
+    return ptr - offset
+
+
+def encode_varint64(value: int) -> bytes:
+    """
+    Encode a signed 64-bit integer using zigzag encoding + varint.
+    Value 0 is reserved for NaN, so all values are shifted by +1.
+
+    Args:
+        value: The signed 64-bit integer to encode
+
+    Returns:
+        Varint-encoded bytes
+
+    Note: This creates a new bytearray. For better performance,
+    use encode_varint64_to_buffer() to write directly to a buffer.
+    """
+    # Zigzag encoding: (value << 1) ^ (value >> 63)
+    # For Python, we need to handle the sign extension properly
+    val = value << 1 if value >= 0 else ((-value - 1) << 1) | 1
+
+    # Reserve value 0 for NaN
+    val += 1
+
+    # Varint encoding
+    result = bytearray()
+    while val > 0x7F:
+        result.append((val & 0x7F) | 0x80)
+        val >>= 7
+    result.append(val & 0xFF)
+
+    return bytes(result)
+
+
+@nb.njit(cache=True, fastmath=True)
+def decode_varint(data: bytes | memoryview, offset: int = 0) -> tuple[int, int]:
+    """
+    Decode a zigzag-encoded varint from bytes.
+    Handles the NaN reservation (value 0 is NaN).
+
+    JIT-compiled with Numba for maximum performance.
+
+    Args:
+        data: The byte buffer
+        offset: Starting offset in the buffer
+
+    Returns:
+        Tuple of (decoded_value, bytes_consumed)
+    """
+    uval = 0
+    shift = 0
+    ptr = offset
+
+    while True:
+        if ptr >= len(data):
+            raise RuntimeError("Incomplete varint in buffer")
+
+        byte = data[ptr]
+        ptr += 1
+        uval |= (byte & 0x7F) << shift
+        shift += 7
+
+        if (byte & 0x80) == 0:
+            break
+
+    # Value 0 is reserved for NaN
+    uval -= 1
+
+    # Zigzag decoding (branchless)
+    val = (uval >> 1) ^ -(uval & 1)
+
+    return val, ptr - offset
+
+
+def encode(value: float, buff: BufferView, format_char: str) -> None:
+    """
+    Encode a primitive value into the buffer using struct.pack.
+
+    Args:
+        value: The value to encode
+        buff: The buffer view to write to
+        format_char: struct format character (e.g., 'f' for float, 'I' for uint32)
+    """
+    data = struct.pack(f"<{format_char}", value)  # Little-endian
+    buff.write_bytes(data)
+
+
+def encode_string(s: str, buff: BufferView) -> None:
+    """
+    Encode a string as uint16 length + UTF-8 bytes.
+
+    Args:
+        s: The string to encode
+        buff: The buffer view to write to
+    """
+    encoded = s.encode("utf-8")
+    length = len(encoded)
+    if length > 65535:
+        raise ValueError(f"String too long: {length} bytes (max 65535)")
+
+    # Write length as uint16
+    encode(length, buff, "H")
+    # Write string bytes
+    buff.write_bytes(encoded)
+
+
+def decode(buff: BufferView, format_char: str) -> int | float:
+    """
+    Decode a primitive value from the buffer using struct.unpack.
+
+    Args:
+        buff: The buffer view to read from
+        format_char: struct format character
+
+    Returns:
+        The decoded value
+    """
+    size = struct.calcsize(f"<{format_char}")
+    data = buff.read_bytes(size)
+    return struct.unpack(f"<{format_char}", data)[0]
+
+
+def decode_string(buff: BufferView) -> str:
+    """
+    Decode a string (uint16 length + UTF-8 bytes).
+
+    Args:
+        buff: The buffer view to read from
+
+    Returns:
+        The decoded string
+    """
+    length = int(decode(buff, "H"))
+    encoded = buff.read_bytes(length)
+    return encoded.decode("utf-8")
+
+
+def to_int64(data: bytes | bytearray | memoryview, offset: int, dtype: str) -> int:
+    """
+    Read a value from bytes and convert to int64.
+
+    Args:
+        data: The byte buffer (bytes, bytearray, or memoryview)
+        offset: Offset in the buffer
+        dtype: Data type ('b', 'B', 'h', 'H', 'i', 'I', 'q', 'Q')
+
+    Returns:
+        The value as int64
+    """
+    value = struct.unpack_from(f"<{dtype}", data, offset)[0]
+    return int(value)
+
+
+def build_field_metadata(
+    info: EncodingInfo,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Build numpy arrays for field metadata to pass to JIT functions.
+
+    Creates three arrays:
+    - field_offsets: byte offset of each field
+    - field_types: type code for each field
+    - field_resolutions: resolution for each field
+
+    Resolution sentinel values:
+      > 0.0  -> lossy quantize+delta+varint
+      == 0.0 -> XOR lossless (FLOAT64 only)
+      -1.0   -> raw 4-byte copy (FLOAT32 without resolution)
+
+    Args:
+        info: Encoding configuration
+
+    Returns:
+        Tuple of (field_offsets, field_types, field_resolutions)
+    """
+    num_fields = len(info.fields)
+
+    field_offsets = np.zeros(num_fields, dtype=np.int32)
+    field_types = np.zeros(num_fields, dtype=np.int32)
+    field_resolutions = np.zeros(num_fields, dtype=np.float64)
+
+    for idx, field in enumerate(info.fields):
+        field_offsets[idx] = field.offset
+        field_types[idx] = int(field.type)
+
+        if info.encoding_opt == EncodingOptions.LOSSY and field.resolution is not None:
+            field_resolutions[idx] = field.resolution
+        elif field.type == FieldType.FLOAT64:
+            field_resolutions[idx] = 0.0
+        elif field.type == FieldType.FLOAT32:
+            field_resolutions[idx] = -1.0
+        else:
+            field_resolutions[idx] = 0.0
+
+    return field_offsets, field_types, field_resolutions