pycellpack 0.1.0__tar.gz

pycellpack-0.1.0/Cargo.lock

@@ -0,0 +1,372 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "bitflags"
+version = "2.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
+
+[[package]]
+name = "cellpack"
+version = "0.1.0"
+dependencies = [
+ "crossbeam-queue",
+ "crossbeam-utils",
+ "io-uring",
+ "libc",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "crossbeam-queue"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "io-uring"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "numpy"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4"
+dependencies = [
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
+dependencies = [
+ "portable-atomic",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pycellpack"
+version = "0.1.0"
+dependencies = [
+ "cellpack",
+ "numpy",
+ "pyo3",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "thiserror"
+version = "2.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

pycellpack-0.1.0/Cargo.toml

@@ -0,0 +1,35 @@
+[workspace]
+members = ["crates/cellpack", "crates/pycellpack"]
+resolver = "3"
+
+[workspace.package]
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.89"
+license = "MIT"
+publish = false
+repository = "https://github.com/CladeTeam/CellPack"
+homepage = "https://cladeteam.github.io/CellPack/"
+
+[workspace.dependencies]
+thiserror = "2"
+libc = "0.2"
+rustix = { version = "1", features = ["fs", "io_uring", "mm"] }
+pyo3 = "0.29"
+
+[workspace.lints.rust]
+unsafe_op_in_unsafe_fn = "deny"
+
+[workspace.lints.clippy]
+undocumented_unsafe_blocks = "deny"
+
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+debug = "line-tables-only"
+
+# Release code with full debug info for perf.
+[profile.profiling]
+inherits = "release"
+debug = true

pycellpack-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 wangzhongqi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pycellpack-0.1.0/PKG-INFO

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: pycellpack
+Version: 0.1.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Rust
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: Typing :: Typed
+Requires-Dist: numpy>=2.0
+Requires-Dist: typing-extensions>=4.0
+Requires-Dist: anndata==0.13.4 ; python_full_version >= '3.12' and extra == 'anndata'
+Requires-Dist: h5py>=3.8 ; extra == 'anndata'
+Requires-Dist: scipy>=1.13 ; extra == 'anndata'
+Requires-Dist: maturin>=1.8,<2 ; extra == 'dev'
+Requires-Dist: pytest>=8.0 ; extra == 'dev'
+Requires-Dist: pyright>=1.1.410 ; extra == 'dev'
+Requires-Dist: ruff>=0.12 ; extra == 'dev'
+Requires-Dist: scipy>=1.13 ; extra == 'dev'
+Requires-Dist: pandas>=2 ; extra == 'dev'
+Requires-Dist: anndata==0.13.4 ; python_full_version >= '3.12' and extra == 'dev'
+Requires-Dist: h5py>=3.8 ; extra == 'dev'
+Requires-Dist: scipy>=1.13 ; extra == 'scipy'
+Provides-Extra: anndata
+Provides-Extra: dev
+Provides-Extra: scipy
+License-File: LICENSE
+Summary: CellPack matrix storage, native loading, and AnnData H5AD integration
+License-Expression: MIT
+Requires-Python: >=3.10, <3.16
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Documentation, https://cladeteam.github.io/CellPack/python/
+Project-URL: Homepage, https://cladeteam.github.io/CellPack/
+Project-URL: Issues, https://github.com/CladeTeam/CellPack/issues
+Project-URL: Repository, https://github.com/CladeTeam/CellPack
+
+<p align="center">
+  <img src="assets/logo.svg" alt="CellPack logo" width="128" height="128">
+</p>
+
+<h1 align="center">CellPack</h1>
+
+<p align="center">
+  <strong>Compact single-cell storage. Fast random access. Ready for AnnData.</strong>
+</p>
+
+<p align="center">
+  <a href="https://cladeteam.github.io/CellPack/">Website</a> ·
+  <a href="https://github.com/CladeTeam/CellPack/issues">Issues</a> ·
+  <a href="LICENSE">MIT License</a>
+</p>
+
+<p align="center">
+  <img alt="Python 3.10–3.15" src="https://img.shields.io/badge/python-3.10–3.15-1E7A68?logo=python&logoColor=white">
+  <img alt="Rust core" src="https://img.shields.io/badge/core-Rust-1E7A68?logo=rust">
+  <img alt="Platform: Linux" src="https://img.shields.io/badge/platform-Linux-1E7A68?logo=linux&logoColor=white">
+</p>
+
+CellPack stores expression matrices as compact, independently decodable cell
+records. Read individual cells or shuffled batches without decoding their
+neighbours, work with datasets larger than memory, and keep your AnnData
+annotations alongside the compressed matrices.
+
+The storage engine and loader are written in Rust. **`pycellpack`** brings them
+to Python, with NumPy, SciPy, AnnData and PyTorch integrations.
+
+## What you can do
+
+- **Compress without losing values.** Shared statistical models capture sparse
+  matrix structure while each cell remains independently decodable.
+- **Read just the cells and genes you need.** Lazy matrix views support slicing,
+  masks and reordered selections; materialize the selected data as dense or CSR.
+- **Keep annotated datasets together.** H5CP embeds CellPack matrices in one
+  HDF5 file alongside AnnData annotations, layers, embeddings and metadata.
+- **Load shuffled data beyond RAM.** Asynchronous I/O overlaps reads and
+  decoding; bounded buffers deliver batches in the requested sampling order.
+- **Make reuse count.** A planner uses the supplied access sequence to allocate
+  a cache budget between compressed pages and decoded rows.
+- **Feed model inputs directly.** Align genes across datasets, apply native
+  preprocessing and deliver PyTorch batches on CPU or CUDA. Presets cover
+  scGPT, Geneformer, UCE and STATE SE.
+
+## AnnData, with compressed matrices
+
+Start with an ordinary AnnData object, including one from an existing Scanpy
+workflow. Write it once, then select cells and genes before decoding:
+
+```python
+import anndata as ad
+import numpy as np
+import pycellpack as cp
+
+adata = ad.AnnData(X=np.array([[0, 2, 0], [1, 0, 3]], dtype=np.float32))
+adata.obs_names = ["cell-a", "cell-b"]
+adata.var_names = ["ACTB", "GAPDH", "CD3D"]
+adata.layers["counts"] = adata.X.copy()
+
+# One portable file: compressed matrices and their annotations.
+cp.write_h5cp("cells.h5cp", adata, encode=True)
+cells = cp.read_h5("cells.h5cp", lazy=True)
+
+# AnnData slicing keeps the embedded matrices lazy.
+subset = cells[["cell-b"], ["CD3D", "ACTB"]]
+small = subset.to_memory()
+print(small.X)  # [[3. 1.]]
+
+# Export an ordinary H5AD for downstream tools.
+cp.write_h5ad("subset.h5ad", small)
+```
+
+`encode=True` compresses numeric matrix slots; `lazy=True` keeps embedded
+matrices on disk while annotations are loaded into memory. Decode a subset
+with `to_memory()` before passing it to tools that expect NumPy or SciPy
+matrices. H5CP is read through `pycellpack`; `write_h5ad()` exports standard H5AD.
+
+For training, `ScDataset.from_anndata(cells, ids=gene_ids)` connects a lazy
+matrix to `ScDataLoader`. Supply one vocabulary ID per gene and your own sample
+order; the loader handles alignment, preprocessing and batch delivery.
+
+## Performance
+
+The figures below are newly drawn from frozen experimental results. They
+describe the experimental implementation used for the technical report,
+rather than benchmarks of the current commit.
+
+### Compact storage
+
+![Compression ratios for CellPack and selected general-purpose codecs; larger is better.](assets/performance-compression.svg)
+
+CellPack reaches **5.89× compression** while retaining per-cell decoding.
+Ratios are means across raw and normalized matrix blocks: 734 for CellPack and
+724 for the other codecs. These are corpus summaries, not a paired-input
+comparison or a ranking of decoding speed.
+
+### Random-order loading
+
+![Global random-order loading on one NVMe SSD with 16 workers; a logarithmic axis shows cells per second.](assets/performance-loading.svg)
+
+On one NVMe SSD with 16 workers, the report gives **1.165M cells/s** for
+CellPack without a data cache. Baselines use a **20 GB cache** and their best
+variant, averaged over three seeds. CellPack's point is the report's
+author-selected value for a 5 s post-warm-up check, without a variance estimate;
+baselines use 60 s measurement windows. CellPack emits a fixed 2,048-gene panel;
+baselines use their sample-output paths. Only global random-order methods are
+shown.
+
+The [figure data and provenance](assets/performance.json) record the exact
+values and source hashes. Both figures can be regenerated with
+`uv run scripts/plot_readme.py`.
+
+## Install
+
+From a repository checkout, with a Rust toolchain installed:
+
+```sh
+# Python 3.12+ for AnnData integration (pinned to AnnData 0.13.4).
+uv add '/path/to/CellPack[anndata]'
+
+# Or install the matrix storage and loader core (Python 3.10–3.15).
+uv add /path/to/CellPack
+```
+
+CellPack targets Linux. The default loader uses `io_uring`; a `pread` backend
+is available where `io_uring` is disabled. Install PyTorch separately for
+`ScDataLoader`, choosing the build appropriate for your device.
+
+## License
+
+[MIT](LICENSE).
+