lfpack 0.1.0__tar.gz

lfpack-0.1.0/.githooks/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -e
+uv run ruff check src/ tests/
+uv run ruff format --check src/ tests/

lfpack-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,57 @@
+name: docs
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv python install 3.14
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Build API reference
+        run: uv run quartodoc build --config docs/_quarto.yml
+
+      - name: Render site
+        run: quarto render docs/
+
+      - name: Upload pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

lfpack-0.1.0/.github/workflows/publish_to_pypi.yaml

@@ -0,0 +1,26 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

lfpack-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,51 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv python install 3.14
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Run tests with coverage
+        run: uv run pytest tests/ -v --cov=lfpack --cov-report=term-missing --cov-report=json
+
+      - name: Post coverage to step summary
+        if: always()
+        run: |
+          python -c "
+          import json, pathlib, os
+          data = json.loads(pathlib.Path('coverage.json').read_text())
+          t = data['totals']
+          lines = [
+            '## Coverage report',
+            f\"| Statements | Covered | Missed | % |\",
+            f\"|---|---|---|---|\",
+            f\"| {t['num_statements']} | {t['covered_lines']} | {t['missing_lines']} | **{t['percent_covered']:.1f}%** |\",
+            '',
+            '| File | Stmts | Miss | Cover |',
+            '|---|---|---|---|',
+          ]
+          for path, info in data['files'].items():
+            s = info['summary']
+            lines.append(f\"| \`{path}\` | {s['num_statements']} | {s['missing_lines']} | {s['percent_covered']:.1f}% |\")
+          summary = os.environ.get('GITHUB_STEP_SUMMARY', '/dev/null')
+          with open(summary, 'a') as f:
+            f.write('\n'.join(lines) + '\n')
+          "

lfpack-0.1.0/.gitignore

@@ -0,0 +1,134 @@
+TODO.md
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# Quarto rendered output
+docs/_site/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

lfpack-0.1.0/CHANGELOG.md

@@ -0,0 +1,6 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

lfpack-0.1.0/CLAUDE.md

@@ -0,0 +1,77 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this project is
+
+**lfpack** is a lossy LFP (local-field-potential) codec for Neuropixels recordings. It compresses raw `.cbin` LFP binaries into HDF5 files at >100× compression with RMSE < 25 µV, using an 8-stage pipeline followed by adaptive SVD + wavelet-packet thresholding.
+
+## Commands
+
+```bash
+# Dev setup
+uv sync --group dev
+git config core.hooksPath .githooks   # installs ruff pre-commit hook
+
+# Tests
+uv run pytest                          # all tests
+uv run pytest tests/test_lfpack.py::TestLfpack::test_compress_output_shapes  # single test
+
+# Lint / format
+uv run ruff check src/ tests/
+uv run ruff format src/ tests/
+
+# Docs — regenerate API reference from docstrings, then preview
+uv run quartodoc build --config docs/_quarto.yml
+quarto preview docs/
+```
+
+## Architecture
+
+Nearly all code lives in `src/lfpack/_core.py` (~1000 lines). The public API is re-exported from `src/lfpack/__init__.py`.
+
+### Compression pipeline (`compress_bin_to_h5`)
+
+1. Bad-channel detection (`ibldsp.voltage.detect_bad_channels_cbin`)
+2. Dephasing — sample-shift correction (NP1 only, via `ibldsp.fourier.fshift`)
+3. Highpass filter — 2 Hz zero-phase 3rd-order Butterworth
+4. Bad-channel interpolation — distance-weighted neighbors
+5. CAR — median subtraction across channels
+6. Decimation — 2500 → 250 Hz (Q=10)
+7. Cadzow denoising — spatial rank reduction (`ibldsp.cadzow`)
+8. **Adaptive SVD + wavelet-packet thresholding** — the actual codec
+
+Steps 6–7 are checkpointed to a `.npy` file so the expensive decimation can be skipped on resume. Steps 7–8 are parallelised with `joblib`.
+
+### Codec (`compress` / `decompress`)
+
+`compress(data, epsilon=150, alpha=28)` → `LFPCompressed` dataclass:
+- **SVD rank selection**: keep singular values > `epsilon × sigma_noise`
+- **Wavelet-packet thresholding**: db4, level 5; per-component threshold `alpha × sigma_noise / sv[k]`
+- Sparse Vh stored as `(vh_indices, vh_values)` in HDF5; `U_scaled` with shuffle+gzip
+
+Guard bands: 64-sample Cadzow halos, 128-sample SVD/WP overlap to prevent edge transients.
+
+### HDF5 layout (multi-recording, pyramidal)
+
+```
+<file>.h5
+└─ <recording>/
+   └─ <scale_2digit>/
+      ├─ meta         # nc, ns_total, fs, fs_sync, t0_sync, epsilon, alpha, geometry, …
+      └─ chunks/
+         └─ <i>/      # U_scaled, vh_indices, vh_values + attrs
+```
+
+Legacy flat layout (meta at root) is still readable for backwards compatibility.
+
+### `LFPackReader`
+
+Drop-in replacement for `spikeglx.Reader`. Wraps an HDF5 file and decompresses chunks on demand. Supports multi-recording files via `recording=` kwarg and pyramidal scales via `scale=` kwarg.
+
+## Ecosystem interconnections
+
+lfpack sits in a tight three-way dependency with two sibling IBL packages:
+
+- **ibl-neuropixel** (`/Users/olivier/PycharmProjects/ephys-atlas/ibl-neuropixel`) — provides the low-level Neuropixels binary I/O (`spikeglx.Reader`), destriping, dephasing (`ibldsp.fourier.fshift`), decimation (`ibldsp.voltage.resample_denoise_lfp_cbin`), bad-channel detection (`ibldsp.voltage.detect_bad_channels_cbin`), and Cadzow denoising (`ibldsp.cadzow`). lfpack wraps these directly; changes to ibldsp can break the pipeline.
+- **viewephys** (`/Users/olivier/PycharmProjects/ephys-atlas/viewephys`) — Qt-based interactive viewer for raw Neuropixels traces. `LFPackReader` is a drop-in for `spikeglx.Reader`, so any file readable by viewephys can be replaced with an `LFPackReader` instance. `viewephys(traces.T, fs=sr.fs)` is the canonical way to visualise decompressed LFP data; note the transpose (`LFPackReader` returns time-first, viewephys expects channel-first).

lfpack-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,77 @@
+# Contributing to lfpack
+
+## Development setup
+
+```bash
+git clone https://github.com/int-brain-lab/lfpack
+cd lfpack
+uv sync --group dev
+git config core.hooksPath .githooks
+```
+
+The last command installs the pre-commit hook from the tracked `.githooks/` directory.
+
+## Code style
+
+All code is formatted and linted with [ruff](https://docs.astral.sh/ruff/).
+Configuration lives in `pyproject.toml` (`line-length = 120`, rules `E F W I`).
+
+Before committing, run:
+
+```bash
+uv run ruff format src/ tests/   # format
+uv run ruff check  src/ tests/   # lint
+```
+
+The pre-commit hook installed above runs both checks automatically and blocks commits that fail.
+
+## Docstrings
+
+All public functions, classes, and methods use [NumPy/SciPy docstring format](https://numpydoc.readthedocs.io/en/latest/format.html).
+These docstrings are rendered into the API reference by [quartodoc](https://machow.github.io/quartodoc/).
+
+## Documentation
+
+The docs site lives in `docs/` and follows the [Diátaxis](https://diataxis.fr) framework:
+
+| Directory | Type | Purpose |
+|---|---|---|
+| `docs/tutorials/` | Tutorial | Guided learning paths |
+| `docs/how-to/` | How-To | Step-by-step task guides |
+| `docs/reference/` | Reference | API docs (quartodoc) + HDF5 spec |
+| `docs/explanation/` | Explanation | Concepts, design choices, benchmarks |
+
+Build the docs:
+
+```bash
+# Generate API reference pages from docstrings
+uv run quartodoc build --config docs/_quarto.yml
+
+# Render HTML (output → docs/_site/)
+quarto render docs/
+
+# Live preview with hot-reload
+quarto preview docs/
+```
+
+## Tests
+
+```bash
+uv run pytest                     # run all tests
+uv run pytest --cov --cov-report=term-missing   # with coverage report
+```
+
+All existing tests must pass and coverage must not regress before opening a pull request.
+
+## Versioning
+
+This project uses [Semantic Versioning](https://semver.org):
+
+- **PATCH** (`0.1.x`) — bug fixes, no API changes.
+- **MINOR** (`0.x.0`) — new backwards-compatible features or API additions.
+- **MAJOR** (`x.0.0`) — breaking changes (e.g. new HDF5 layout, removed parameters).
+
+Changes are recorded in `CHANGELOG.md` following the
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) convention.
+The `[Unreleased]` section accumulates work until a release, at which point it is
+renamed to `[x.y.z] - YYYY-MM-DD` and a matching git tag `vx.y.z` is pushed.

lfpack-0.1.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: lfpack
+Version: 0.1.0
+Summary: LFP codec — lossy compression of local-field-potential recordings via adaptive SVD and wavelet-packet thresholding
+Project-URL: Homepage, https://github.com/int-brain-lab/lfpack
+Project-URL: Bug Tracker, https://github.com/int-brain-lab/lfpack/issues
+Author: The International Brain Laboratory
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: h5py>=3.0
+Requires-Dist: ibl-neuropixel
+Requires-Dist: numpy
+Requires-Dist: pywavelets
+Requires-Dist: scipy
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# lfpack — LFP codec for Neuropixels recordings
+
+<p align="center">
+  <img src="docs/figures/logo.png" alt="lfpack logo" width="300"/>
+</p>
+
+Lossy codec for local-field-potential (LFP) recordings from Neuropixels probes.
+Achieves **>100× compression** with median RMSE < 25 µV via an 8-stage pipeline
+(bad-channel detection → dephasing → highpass → interpolation → CAR → decimation → Cadzow → adaptive SVD + wavelet-packet thresholding).
+
+```bash
+pip install lfpack
+```
+
+## Documentation
+
+Full documentation is at **https://int-brain-lab.github.io/lfpack/**.
+
+| Section | Contents |
+| --- | --- |
+| [Tutorial](https://int-brain-lab.github.io/lfpack/tutorials/first-compression.html) | End-to-end compression and decompression of a recording |
+| [How-To: binned reads](https://int-brain-lab.github.io/lfpack/how-to/binned-reads.html) | Memory-efficient channel-binned access |
+| [How-To: multi-recording files](https://int-brain-lab.github.io/lfpack/how-to/multi-recording.html) | Combining multiple recordings in one HDF5 file |
+| [API reference](https://int-brain-lab.github.io/lfpack/reference/) | Full public API (`compress_bin_to_h5`, `LFPackReader`, …) |
+| [HDF5 format](https://int-brain-lab.github.io/lfpack/reference/hdf5-layout.html) | On-disk layout specification |
+| [Pipeline explanation](https://int-brain-lab.github.io/lfpack/explanation/pipeline.html) | Stage-by-stage description of the compression pipeline |
+| [SVD+WP benchmark](https://int-brain-lab.github.io/lfpack/explanation/benchmark.html) | RMSE, SNR, and compression-ratio results across 11 insertions |

lfpack-0.1.0/README.md

@@ -0,0 +1,27 @@
+# lfpack — LFP codec for Neuropixels recordings
+
+<p align="center">
+  <img src="docs/figures/logo.png" alt="lfpack logo" width="300"/>
+</p>
+
+Lossy codec for local-field-potential (LFP) recordings from Neuropixels probes.
+Achieves **>100× compression** with median RMSE < 25 µV via an 8-stage pipeline
+(bad-channel detection → dephasing → highpass → interpolation → CAR → decimation → Cadzow → adaptive SVD + wavelet-packet thresholding).
+
+```bash
+pip install lfpack
+```
+
+## Documentation
+
+Full documentation is at **https://int-brain-lab.github.io/lfpack/**.
+
+| Section | Contents |
+| --- | --- |
+| [Tutorial](https://int-brain-lab.github.io/lfpack/tutorials/first-compression.html) | End-to-end compression and decompression of a recording |
+| [How-To: binned reads](https://int-brain-lab.github.io/lfpack/how-to/binned-reads.html) | Memory-efficient channel-binned access |
+| [How-To: multi-recording files](https://int-brain-lab.github.io/lfpack/how-to/multi-recording.html) | Combining multiple recordings in one HDF5 file |
+| [API reference](https://int-brain-lab.github.io/lfpack/reference/) | Full public API (`compress_bin_to_h5`, `LFPackReader`, …) |
+| [HDF5 format](https://int-brain-lab.github.io/lfpack/reference/hdf5-layout.html) | On-disk layout specification |
+| [Pipeline explanation](https://int-brain-lab.github.io/lfpack/explanation/pipeline.html) | Stage-by-stage description of the compression pipeline |
+| [SVD+WP benchmark](https://int-brain-lab.github.io/lfpack/explanation/benchmark.html) | RMSE, SNR, and compression-ratio results across 11 insertions |

lfpack-0.1.0/docs/.gitignore

@@ -0,0 +1,2 @@
+/.quarto/
+**/*.quarto_ipynb

lfpack-0.1.0/docs/README.txt

@@ -0,0 +1,25 @@
+Local development
+-----------------
+
+1. Install dependencies (once):
+     uv sync --group dev
+
+2. Regenerate API reference from docstrings (run after editing _core.py):
+     uv run quartodoc build --config docs/_quarto.yml
+
+3. Live-preview the site:
+     quarto preview docs/
+
+The preview server reloads automatically when .qmd files change.
+Re-run step 2 manually whenever docstrings change — quartodoc output
+is not watched by the preview server.
+
+
+Production build
+----------------
+
+The site is built and deployed automatically by GitHub Actions on every
+push to main (.github/workflows/docs.yml).  The workflow runs the same
+three steps above (quartodoc build → quarto render → deploy to GitHub Pages).
+
+Do not commit the generated _site/ directory; it is built in CI.

lfpack-0.1.0/docs/_quarto.yml

@@ -0,0 +1,64 @@
+project:
+  type: website
+  output-dir: _site
+
+website:
+  title: "lfpack"
+  favicon: figures/favicon.png
+  repo-url: https://github.com/int-brain-lab/lfpack
+  navbar:
+    left:
+      - text: "Tutorial"
+        file: tutorials/first-compression.qmd
+      - text: "How-To"
+        menu:
+          - text: "Binned-channel reads"
+            file: how-to/binned-reads.qmd
+          - text: "Multi-recording files"
+            file: how-to/multi-recording.qmd
+      - text: "Reference"
+        menu:
+          - text: "API"
+            file: reference/index.qmd
+          - text: "HDF5 format"
+            file: reference/hdf5-layout.qmd
+      - text: "Explanation"
+        menu:
+          - text: "Compression pipeline"
+            file: explanation/pipeline.qmd
+          - text: "Benchmark gallery"
+            file: explanation/gallery.qmd
+    right:
+      - icon: github
+        href: https://github.com/int-brain-lab/lfpack
+
+format:
+  html:
+    theme: minty
+    toc: true
+    code-copy: true
+    code-overflow: wrap
+    lightbox: true
+
+quartodoc:
+  package: lfpack
+  dir: reference
+  sidebar: reference/_sidebar.yml
+  sections:
+    - title: "Compression"
+      desc: "Compress LFP recordings to HDF5."
+      contents:
+        - compress_bin_to_h5
+        - compress_to_h5
+        - compress_pipeline
+        - compress
+        - decompress
+        - run_cadzow_checkpoint
+    - title: "Data types"
+      desc: "Core data structures."
+      contents:
+        - LFPCompressed
+    - title: "Reader"
+      desc: "Random-access decompression."
+      contents:
+        - LFPackReader

lfpack-0.1.0/docs/explanation/gallery.qmd

@@ -0,0 +1,51 @@
+---
+title: "Benchmark gallery"
+---
+
+Compression results across 11 IBL Neuropixels (NP1) recordings spanning a range of brain
+regions and recording quality.  Each recording was compressed with two parameter sets:
+
+| Label | ε | α | Typical CR |
+|---|---|---|---|
+| **default** | 150 | 28 | ~350× |
+| **aggressive** | 450 | 96 | ~750× |
+
+Click any figure to open it full-size with keyboard navigation.
+
+---
+
+## Aggregate metrics
+
+RMSE distribution (bar = median, ▽ = p95) and compression ratio across all recordings.
+
+![Aggregate RMSE and compression ratio across all 11 PIDs](../figures/gallery/2026-06-15_lfp_compression_aggregate.png){.lightbox}
+
+---
+
+## Voltage density
+
+Columns within each panel: original → Cadzow → default → aggressive (top row), with
+residuals vs the previous stage (bottom row).
+
+![`1a276285`](../figures/gallery/2026-06-02_lfp_density_voltage_1a276285-8b0e-4cc9-9f0a-a3a002978724.png){group="density" .lightbox}
+
+![`1e104bf4`](../figures/gallery/2026-06-02_lfp_density_voltage_1e104bf4-7a24-4624-a5b2-c2c8289c0de7.png){group="density" .lightbox}
+
+![`6638cfb3`](../figures/gallery/2026-06-02_lfp_density_voltage_6638cfb3-3831-4fc2-9327-194b76cf22e1.png){group="density" .lightbox}
+
+![`749cb2b7`](../figures/gallery/2026-06-02_lfp_density_voltage_749cb2b7-e57e-4453-a794-f6230e4d0226.png){group="density" .lightbox}
+
+![`d7ec0892`](../figures/gallery/2026-06-02_lfp_density_voltage_d7ec0892-0a6c-4f4f-9d8f-72083692af5c.png){group="density" .lightbox}
+
+![`da8dfec1`](../figures/gallery/2026-06-02_lfp_density_voltage_da8dfec1-d265-44e8-84ce-6ae9c109b8bd.png){group="density" .lightbox}
+
+![`dab512bd`](../figures/gallery/2026-06-02_lfp_density_voltage_dab512bd-a02d-4c1f-8dbc-9155a163efc0.png){group="density" .lightbox}
+
+![`dc7e9403`](../figures/gallery/2026-06-02_lfp_density_voltage_dc7e9403-19f7-409f-9240-05ee57cb7aea.png){group="density" .lightbox}
+
+![`e8f9fba4`](../figures/gallery/2026-06-02_lfp_density_voltage_e8f9fba4-d151-4b00-bee7-447f0f3e752c.png){group="density" .lightbox}
+
+![`eebcaf65`](../figures/gallery/2026-06-02_lfp_density_voltage_eebcaf65-7fa4-4118-869d-a084e84530e2.png){group="density" .lightbox}
+
+![`fe380793`](../figures/gallery/2026-06-02_lfp_density_voltage_fe380793-8035-414e-b000-09bfe5ece92a.png){group="density" .lightbox}
+