annslicer 0.1.3__tar.gz

annslicer-0.1.3/.github/workflows/benchmark.yml

@@ -0,0 +1,26 @@
+name: Benchmark
+
+on:
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: Benchmark (Python 3.10)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch full history so setuptools-scm can derive a version from tags.
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: pip
+
+      - name: Install package with dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run benchmarks
+        run: make benchmark

annslicer-0.1.3/.github/workflows/build.yml

@@ -0,0 +1,31 @@
+name: Build check
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    name: Build sdist and wheel
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # setuptools-scm needs full history + tags
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: pip
+
+      - name: Install build tools
+        run: pip install --upgrade pip build twine
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Check distributions
+        run: twine check dist/*

annslicer-0.1.3/.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    name: ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install ruff and mypy
+        run: pip install ruff mypy==1.15.0
+
+      - name: Check (lint)
+        run: ruff check src/ tests/
+
+      - name: Check (format)
+        run: ruff format --check src/ tests/
+
+      - name: Type-check (mypy)
+        run: mypy src/annslicer

annslicer-0.1.3/.github/workflows/publish.yml

@@ -0,0 +1,51 @@
+name: Publish to PyPI
+
+# Runs automatically when a GitHub Release is published, AND can be
+# triggered manually from the Actions tab as a fallback.
+#
+# Before creating a release:
+#   git tag v0.x.y
+#   git push --tags
+#   git push origin --tags
+# Then: GitHub → Releases → Draft a new release → choose the tag → Publish.
+#
+# PyPI Trusted Publishing (OIDC) is used — no API token stored in secrets.
+# One-time setup on PyPI: add a "Trusted Publisher" for this repo/workflow.
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi  # must match the environment name set in PyPI trusted publisher config
+
+    # Required for PyPI Trusted Publishing (OIDC token exchange)
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Full history so setuptools-scm picks up the tag
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install build tools
+        run: pip install --upgrade pip build twine
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Check distributions
+        run: twine check dist/*
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # No username/password needed — OIDC handles authentication.

annslicer-0.1.3/.github/workflows/test.yml

@@ -0,0 +1,33 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: pytest (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch full history so setuptools-scm can derive a version from tags.
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install package with dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest

annslicer-0.1.3/.gitignore

@@ -0,0 +1,40 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+*.egg
+.eggs/
+MANIFEST
+
+# setuptools-scm generated version file
+src/annslicer/_version.py
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Pytest
+.pytest_cache/
+.cache/
+
+# Ruff
+.ruff_cache/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# macOS
+.DS_Store
+
+# Large data files — never commit .h5ad files to the repo
+*.h5ad
+*.h5
+*.hdf5

annslicer-0.1.3/CONTRIBUTING.md

@@ -0,0 +1,132 @@
+# Contributing to annslicer
+
+## Development setup
+
+We recommend working inside a conda environment to keep dependencies isolated.
+
+```bash
+conda create -n annslicer python=3.10
+conda activate annslicer
+```
+
+Clone the repo:
+
+```bash
+git clone https://github.com/cellarium-ai/annslicer.git
+cd annslicer
+```
+
+And then install in editable mode with development dependencies. You can then either install using the Makefile command
+
+```bash
+make install
+```
+
+or instead, you can equivalently run
+
+```bash
+pip install --upgrade pip
+pip install -e ".[dev]"
+```
+
+The `annslicer` command will now point to your local source, and changes take effect immediately without reinstalling.
+
+## Running tests
+
+```bash
+pytest
+```
+
+Zarr-related tests (zarr output merging, zarr input slicing, zarr shuffle) are skipped automatically if `zarr` is not installed.
+To run the full test suite including zarr:
+
+```bash
+pip install -e ".[dev,zarr]"
+pytest
+```
+
+## Linting, formatting, and type-checking
+
+Before committing, run two commands from the root of the repo:
+
+```bash
+make lint
+make typecheck
+```
+
+Or if you want to type things manually:
+
+```bash
+# Check for lint errors
+ruff check src/ tests/
+
+# Auto-fix where possible
+ruff check --fix src/ tests/
+
+# Check formatting
+ruff format --check src/ tests/
+
+# Apply formatting
+ruff format src/ tests/
+
+# Type-check
+mypy src/annslicer
+```
+
+All three checks run automatically on every push and pull request via GitHub Actions.
+
+## Running benchmarks
+
+Benchmarks live in `benchmarks/` and are excluded from the normal `pytest` run so that CI stays fast. Run them locally with:
+
+```bash
+make benchmark
+```
+
+Or directly:
+
+```bash
+pytest benchmarks/ --benchmark-only -v
+```
+
+The benchmark suite (`benchmarks/bench_slice.py`) compares:
+
+| Benchmark | What it measures |
+|---|---|
+| `bench_annslicer_slice` | Full out-of-core sharding pipeline (no shuffle) |
+| `bench_annslicer_slice_shuffle` | Overhead of random shuffling via sort-read-reorder |
+| `bench_anndata_backed_iterate` | Baseline: backed AnnData row iteration |
+
+Adjust `N_CELLS_BENCH` and `N_GENES_BENCH` in `benchmarks/conftest.py` to scale the dataset up or down.
+
+## Releasing a new version
+
+Version is derived automatically from git tags — there is no version string to update in code.
+
+1. Ensure all tests and lint checks pass on `main`.
+2. Tag the release commit:
+   ```bash
+   git tag v0.2.0
+   git push --tags
+   ```
+3. In the GitHub Actions tab, manually trigger the **Publish to PyPI** workflow.
+
+That's it. `setuptools-scm` picks the version from the tag, builds the sdist and wheel, and publishes to PyPI using OIDC Trusted Publishing (no API token required).
+
+## Project layout
+
+```
+src/annslicer/
+    __init__.py   — public API surface and __version__
+    cli.py        — unified entry point; registers slice and merge subcommands
+    _store.py     — shared HDF5/Zarr store helpers (open_store, _is_sparse_group, _require_zarr)
+    slice.py      — shard logic: shard_h5ad (h5ad + zarr input, shuffle), register_subcommand
+    merge.py      — merge logic: merge_out_of_core (h5ad + zarr output), register_subcommand
+tests/
+    conftest.py   — synthetic .h5ad and .zarr fixtures
+    test_slice.py — unit and integration tests for slicing (including shuffle + zarr input)
+    test_merge.py — unit and integration tests for merging (h5ad + zarr)
+benchmarks/
+    conftest.py   — large synthetic fixture (configurable N_CELLS_BENCH × N_GENES_BENCH)
+    bench_slice.py — annslicer slice vs. backed AnnData iteration benchmarks
+```

annslicer-0.1.3/LICENSE.md

@@ -0,0 +1,26 @@
+Copyright (c) 2026, Broad Institute
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

annslicer-0.1.3/Makefile

@@ -0,0 +1,23 @@
+.PHONY: install lint typecheck benchmark build-check
+
+install:
+	pip install --upgrade pip
+	pip install -e ".[dev]"
+
+lint:
+	ruff check --fix src/ tests/ benchmarks/
+	ruff format src/ tests/ benchmarks/
+
+typecheck:
+	mypy src/annslicer
+
+benchmark:
+	pytest benchmarks/ --benchmark-only -v -s
+
+build-check:
+	@echo "--- Building sdist and wheel ---"
+	python -m build --outdir /tmp/annslicer-dist-check
+	@echo "--- Checking distributions ---"
+	twine check /tmp/annslicer-dist-check/*
+	@rm -rf /tmp/annslicer-dist-check
+	@echo "--- Build check passed ---"

annslicer-0.1.3/PKG-INFO

@@ -0,0 +1,187 @@
+Metadata-Version: 2.4
+Name: annslicer
+Version: 0.1.3
+Summary: Out-of-core sharding of large .h5ad AnnData files with minimal memory usage.
+Author: sfleming
+License: MIT
+Project-URL: Homepage, https://github.com/sfleming/annslicer
+Project-URL: Bug Tracker, https://github.com/sfleming/annslicer/issues
+Keywords: anndata,h5ad,bioinformatics,single-cell,genomics
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: anndata>=0.9
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Requires-Dist: h5py>=3.8
+Provides-Extra: dev
+Requires-Dist: annslicer[zarr]; extra == "dev"
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-benchmark>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Provides-Extra: zarr
+Requires-Dist: zarr>=2.10; extra == "zarr"
+Dynamic: license-file
+
+# annslicer
+
+**Out-of-core sharding and merging of large AnnData files with minimal memory usage.**
+
+![Diagram](diagram.png)
+
+Large single-cell datasets stored as `.h5ad` or `.zarr` files can easily exceed available RAM. `annslicer` slices them into manageable shards — and merges them back — without loading full matrices into memory. It uses best practices from `anndata` with a few small speed improvements for random shuffling.
+
+Consolidates best practices into a simple command-line tool.
+
+```bash
+annslicer slice input.h5ad output_prefix
+```
+
+```bash
+annslicer merge output.h5ad shard_0.h5ad shard_1.h5ad
+```
+
+## Features
+
+- Shards and merges `X`, all `layers`, `obs`, `var`, `obsm`, and `uns`
+- Handles both dense and sparse (CSR) matrices
+- Constant, low memory footprint regardless of file size
+- Input supports both `.h5ad` and `.zarr` formats for slicing
+- Merge output supports both `.h5ad` and `.zarr` formats
+- Optional **cell shuffling** (`--shuffle`) for representative shards without loading the full matrix
+- Simple CLI and Python API
+
+## Installation
+
+```bash
+pip install annslicer
+```
+
+For Zarr input/output support (optional):
+
+```bash
+pip install annslicer[zarr]
+```
+
+## CLI Usage
+
+`annslicer` provides two subcommands: `slice` and `merge`.
+
+### Sharding a large file
+
+```bash
+annslicer slice input.h5ad output_prefix --size 10000
+```
+
+Both `.h5ad` and `.zarr` inputs are supported.
+
+| Argument | Description |
+|---|---|
+| `input.h5ad` or `input.zarr` | Path to the source file |
+| `output_prefix` | Prefix for output files (e.g. `atlas` → `atlas_shard001.h5ad`, …) |
+| `--size N` | Number of cells per shard (default: `10000`) |
+| `--shuffle` | Randomly assign cells to shards (each shard is a representative draw) |
+| `--seed N` | Random seed for reproducible shuffling (requires `--shuffle`) |
+
+**Example — basic sharding:**
+
+```bash
+annslicer slice /data/large_atlas.h5ad /outputs/atlas --size 20000
+```
+
+**Example — shuffled sharding from a large h5ad:**
+
+```bash
+annslicer slice /data/large_atlas.h5ad /outputs/atlas --size 10000 --shuffle --seed 0
+```
+
+Produces: `atlas_shard_0.h5ad`, `atlas_shard_1.h5ad`, …
+
+### Merging shards back into one file
+
+```bash
+annslicer merge output.h5ad shard_0.h5ad shard_1.h5ad shard_2.h5ad
+```
+
+Output format is inferred from the extension — use `.zarr` for Zarr output (requires `annslicer[zarr]`):
+
+```bash
+annslicer merge output.zarr shard_0.h5ad shard_1.h5ad shard_2.h5ad
+```
+
+### Global options
+
+| Flag | Description |
+|---|---|
+| `--debug` | Enable verbose debug-level logging |
+
+## Python API
+
+```python
+from annslicer import shard_h5ad, merge_out_of_core
+
+# Basic sharding (h5ad or zarr input)
+shard_h5ad("large_atlas.h5ad", "atlas", shard_size=20000)
+shard_h5ad("large_atlas.zarr", "atlas", shard_size=20000)  # requires annslicer[zarr]
+
+# Shuffled sharding — cells are randomly distributed across shards
+shard_h5ad("large_atlas.h5ad", "atlas", shard_size=20000, shuffle=True, seed=0)
+
+# Merge shards back into one file
+merge_out_of_core(["atlas_shard_0.h5ad", "atlas_shard_1.h5ad"], "merged.h5ad")
+```
+
+## How it works
+
+### Slicing
+1. Opens the input file ("backed" AnnData for `.h5ad`; `anndata.io.sparse_dataset` for `.zarr`).
+2. If `shuffle=True`, generates a global cell permutation upfront using `numpy.random.default_rng`.
+3. For each shard, reads only the relevant rows from `X` and each layer via sorted fancy indexing — no full matrix is ever loaded into memory.
+4. When shuffling, rows are read in sorted index order (maximising sequential I/O) and then reordered in-memory to the desired shuffled order.
+5. Reassembles a valid `AnnData` object per shard and writes it to disk.
+
+### Merging
+1. Reads `obs`, `var`, and `uns` from the shards to build a skeleton output file.
+2. Scans shards to calculate total non-zero sizes for pre-allocation.
+3. Streams `X`, layers, and `obsm` data shard-by-shard directly into the pre-allocated output arrays.
+
+> **Note:** CSC (column-compressed) sparse matrices are not supported for out-of-core row-slicing. Convert to CSR before sharding.
+
+## Benchmarks
+
+Run on a dummy sparse anndata object with 200k cells and 10k genes.
+
+### For h5ad format
+
+| Slicing method | Mean runtime (s) | Peak memory (MB) |
+|---|---|---|
+| `annslicer slice` | 0.584 | 211.4 |
+| `anndata` backed | 0.601 | 203.7 |
+| `annslicer slice --shuffle` | 1.731 | 221.8 |
+| `anndata` backed with shuffle | 3.830 | 209.1 |
+
+### For zarr format
+
+| Slicing method | Mean runtime (s) | Peak memory (MB) |
+|---|---|---|
+| `annslicer slice` | 1.050 | 62.1 |
+| `anndata` backed | 0.799 | 54.4 |
+| `annslicer slice --shuffle` | 5.544 | 142.9 |
+| `anndata` backed with shuffle | 6.591 | 151.4 |
+
+Based on these benchmarks, for making randomly shuffled data shards, we recommend using `annslicer slice --shuffle` on an h5ad format file.
+
+## License
+
+BSD 3-clause