floatbungler 0.1.0__tar.gz

floatbungler-0.1.0/.github/workflows/CI.yml

@@ -0,0 +1,185 @@
+# This file is autogenerated by maturin v1.12.6
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+    tags:
+      - "*"
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+          - runner: ubuntu-22.04
+            target: x86
+          - runner: ubuntu-22.04
+            target: aarch64
+          - runner: ubuntu-22.04
+            target: armv7
+          - runner: ubuntu-22.04
+            target: s390x
+          - runner: ubuntu-22.04
+            target: ppc64le
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: auto
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: dist
+
+  musllinux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+          - runner: ubuntu-22.04
+            target: x86
+          - runner: ubuntu-22.04
+            target: aarch64
+          - runner: ubuntu-22.04
+            target: armv7
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: musllinux_1_2
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-musllinux-${{ matrix.platform.target }}
+          path: dist
+
+  windows:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: windows-latest
+            target: x64
+            python_arch: x64
+          - runner: windows-latest
+            target: x86
+            python_arch: x86
+          - runner: windows-11-arm
+            target: aarch64
+            python_arch: arm64
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+          architecture: ${{ matrix.platform.python_arch }}
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-windows-${{ matrix.platform.target }}
+          path: dist
+
+  macos:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: macos-15-intel
+            target: x86_64
+          - runner: macos-latest
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-macos-${{ matrix.platform.target }}
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-sdist
+          path: dist
+
+  release:
+    name: Release
+    environment: pypi
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
+    needs: [linux, musllinux, windows, macos, sdist]
+    permissions:
+      # Use to sign the release artifacts
+      id-token: write
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@v7
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-path: "wheels-*/*"
+      - name: Install uv
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: astral-sh/setup-uv@v7
+      - name: Publish to PyPI
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        run: uv publish 'wheels-*/*'

floatbungler-0.1.0/.github/workflows/bench.yml

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2026 Miikka Koskinen
+#
+# SPDX-License-Identifier: ISC
+
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - uses: astral-sh/setup-uv@v7
+
+      - name: Run benchmark
+        run: |
+          uv run pytest bench/ --benchmark-enable --benchmark-json benchmark.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "pytest"
+          output-file-path: benchmark.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true

floatbungler-0.1.0/.gitignore

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 Miikka Koskinen
+#
+# SPDX-License-Identifier: ISC
+
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+*.dSYM
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version
+
+# Hypothesis
+.hypothesis/
+
+# Claude
+.claude/

floatbungler-0.1.0/CHANGELOG.md

@@ -0,0 +1,68 @@
+<!--
+SPDX-FileCopyrightText: 2025 Miikka Koskinen
+
+SPDX-License-Identifier: ISC
+-->
+
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.1.1] - 2026-03-14
+
+### 🐛 Bug Fixes
+
+- Fix patas for long inputs
+
+### 🚜 Refactor
+
+- Replace `count_leading` with a built-in
+- Replace `count_trailing` with a built-in
+- *(patas)* Simplify diff128 logic
+- *(patas)* Pre-allocate the result vector
+- *(patas)* Simplify code for meaningful bytes
+- *(patas)* Copy the float in the loop
+- *(patas)* Pre-allocate the encoding buffer
+
+### ⚙️ Miscellaneous Tasks
+
+- Update the CI workflow
+- Set up a basic benchmark
+- Keep the benchmark cache available
+- Push benchmark results to GH Pages
+- Add benchmark site to README
+- Add test vectors
+- Run cargo fmt
+- `just fix`
+
+## [0.1.0] - 2026-03-14
+
+### 🚀 Features
+
+- Initial commit
+- Add typestubs for floatbungler.gorilla
+- Add a Chimp128 implementation
+- Add a Chimp implementation
+- Add a Patas implementation
+- Add type stubs for all algorithms
+
+### 🚜 Refactor
+
+- Remove debug prints
+- Deduplicate helper functions
+- Deduplicate the module registration
+
+### 📚 Documentation
+
+- Add a README.md
+- Add license information
+- Generate CHANGELOG.md
+- Update motivation for the project
+
+### ⚙️ Miscellaneous Tasks
+
+- Add project metadata for the package
+- Host the package on GitHub after all
+- Release v0.1.0
+
+<!-- generated by git-cliff -->

floatbungler-0.1.0/CLAUDE.md

@@ -0,0 +1,133 @@
+<!--
+SPDX-FileCopyrightText: 2025 Miikka Koskinen
+
+SPDX-License-Identifier: ISC
+-->
+
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+floatbungler is a Python extension module written in Rust using PyO3 that implements floating-point compression algorithms. The project provides four main compression algorithms:
+- **Gorilla**: Facebook's time-series compression algorithm
+- **Chimp**: Standard Chimp compression algorithm with bin-encoded leading zeros
+- **Chimp128**: Advanced Chimp variant with 128-entry ring buffer and 16384-entry lookup table
+- **Patas**: Byte-aligned compression algorithm with 128-entry ring buffer and lookup table
+
+## Build System
+
+This project uses Maturin to build Python extensions from Rust code. The toolchain uses `uv` for Python package management.
+
+### Build Commands
+
+```bash
+# Build the extension module (development mode)
+maturin develop
+
+# Build release version
+maturin build --release
+
+# Run Python tests
+uv run pytest
+
+# Or using just
+just test
+
+```
+
+## Architecture
+
+### Rust Core (`src/`)
+
+The Rust implementation is organized into modules:
+
+- **`lib.rs`**: PyO3 module definition. Creates the `floatbungler` Python module with submodules. Uses the sys.modules workaround to enable direct imports like `from floatbungler import gorilla`.
+
+- **`gorilla.rs`**: Gorilla compression algorithm implementation
+  - `encode_plain()`/`encode()`: Compresses f64 arrays using XOR-based delta encoding with leading/trailing zero optimization
+  - `decode_plain()`/`decode()`: Decompresses back to f64 arrays
+  - Tracks previous leading/trailing zeros for efficient encoding
+  - Uses helper functions from `bit_utils.rs`
+
+- **`chimp.rs`**: Standard Chimp compression algorithm
+  - Similar to Gorilla but uses bin-encoded leading zeros (3 bits instead of 5 bits)
+  - Uses trailing zero threshold of 6 bits to decide between control codes
+  - Simpler than Chimp128 - no ring buffer, just XOR with previous value
+
+- **`chimp128.rs`**: Advanced Chimp128 compression algorithm
+  - Uses a 128-entry ring buffer and 16384-entry lookup table
+  - Selects best reference value from ring buffer based on trailing zeros
+  - Uses trailing zero threshold of 13 bits (log2(128) + log2(64))
+  - Encodes using bin-encoded leading zeros (3 bits) rather than raw values
+
+- **`patas.rs`**: Patas compression algorithm (byte-aligned variant)
+  - Uses a 128-entry ring buffer and 16384-entry lookup table (similar to Chimp128)
+  - Byte-aligned encoding: uses 16-bit header + variable number of bytes for meaningful bits
+  - Header format: 7 bits for reference index, 3 bits for meaningful bytes count, 6 bits for trailing zeros
+  - Stores meaningful bits as whole bytes rather than individual bits
+  - Uses `bytes` crate for efficient buffer management
+
+- **`bits.rs`**: Low-level bit-oriented I/O
+  - `Bitwrite`: Writes individual bits to a byte buffer
+  - `Bitread`: Reads individual bits from a byte slice
+  - Both support f64 and arbitrary-length u64 operations
+
+- **`bit_utils.rs`**: Utility functions for bit manipulation
+  - `bin_count()` / `bin_encode()` / `bin_decode()`: Bin-based encoding for leading zeros (bins: 0, 8, 12, 16, 18, 20, 22, 24)
+
+### Python Interface (`python/floatbungler/`)
+
+- **`__init__.py`**: Re-exports everything from the compiled Rust extension module
+- **Type stubs (`.pyi` files)**: Type stubs are provided for all compression algorithms (`gorilla.pyi`, `chimp.pyi`, `chimp128.pyi`, `patas.pyi`) to enable IDE autocomplete and static type checking
+
+### Testing
+
+- **Python tests**: `test/test_gorilla.py`, `test/test_chimp.py`, `test/test_chimp128.py`, and `test/test_patas.py` use Hypothesis for property-based testing of the compression codecs. Tests use `numpy.testing.assert_equal` to handle NaN comparisons correctly.
+
+## Key Implementation Details
+
+### PyO3 Module Structure
+
+The project uses PyO3's submodule pattern. All four compression algorithm submodules (`gorilla`, `chimp`, `chimp128`, `patas`) are created as separate PyModules and added to sys.modules to support direct imports (`from floatbungler import gorilla`, `from floatbungler import chimp`, `from floatbungler import chimp128`, `from floatbungler import patas`). This is a workaround for PyO3 issue #759.
+
+### Compression Algorithms
+
+All four algorithms use XOR-based compression:
+1. First value is stored uncompressed
+2. Subsequent values are XORed with a reference (previous value for Gorilla/Chimp, ring buffer entry for Chimp128/Patas)
+3. The XOR result is encoded with leading/trailing zero compression
+
+**Gorilla** uses:
+- 5 bits for leading zero count
+- Direct XOR with previous value
+- Bit-level encoding
+
+**Chimp** uses:
+- Bin-encoded leading zeros (3 bits, covering bins: 0, 8, 12, 16, 18, 20, 22, 24)
+- Trailing zero threshold of 6 bits to select control codes
+- Direct XOR with previous value
+- Bit-level encoding
+
+**Chimp128** uses:
+- A 128-entry ring buffer to find better reference values (more trailing zeros = better compression)
+- A lookup table indexed by the low 14 bits for fast ring buffer lookups
+- Trailing zero threshold of 13 bits (log2(128) + log2(64))
+- Bin-encoded leading zeros (same as Chimp)
+- Bit-level encoding
+
+**Patas** uses:
+- A 128-entry ring buffer and 16384-entry lookup table (similar to Chimp128)
+- Byte-aligned encoding with 16-bit headers
+- Header encodes: 7 bits reference index + 3 bits meaningful bytes + 6 bits trailing zeros
+- Stores meaningful bits rounded up to whole bytes
+- Uses `bytes` crate for buffer management instead of bit-level operations
+
+## Development Notes
+
+- The project requires Python 3.10+
+- Maturin is configured to use `python-source = "python"` to find the Python package
+- All compression algorithms handle NaN values correctly
+- To add new dependencies, use `uv add` instead of editing `pyproject.toml`
+- Helper functions for bit manipulation are centralized in `bit_utils.rs` and shared across all algorithms

floatbungler-0.1.0/Cargo.lock

@@ -0,0 +1,171 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+
+[[package]]
+name = "bytes"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+
+[[package]]
+name = "floatbungler"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "pyo3",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
+
+[[package]]
+name = "libc"
+version = "0.2.175"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "portable-atomic"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a"
+dependencies = [
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

floatbungler-0.1.0/Cargo.toml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2025 Miikka Koskinen
+#
+# SPDX-License-Identifier: ISC
+
+[package]
+name = "floatbungler"
+version = "0.1.0"
+edition = "2021"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "floatbungler"
+crate-type = ["cdylib"]
+
+[dependencies]
+bytes = "1.10.1"
+pyo3 = "0.25.0"