polystore 0.1.0__tar.gz

polystore-0.1.0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2025 Tristan Simas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

polystore-0.1.0/PKG-INFO

@@ -0,0 +1,266 @@
+Metadata-Version: 2.4
+Name: polystore
+Version: 0.1.0
+Summary: Framework-agnostic multi-backend storage abstraction for ML and scientific computing
+Author-email: Tristan Simas <tristan.simas@mail.mcgill.ca>
+License: MIT License
+Project-URL: Homepage, https://github.com/OpenHCSDev/PolyStore
+Project-URL: Documentation, https://polystore.readthedocs.io
+Project-URL: Bug Reports, https://github.com/OpenHCSDev/PolyStore/issues
+Project-URL: Source, https://github.com/OpenHCSDev/PolyStore
+Keywords: storage,backend,multi-framework,numpy,pytorch,jax,tensorflow,zarr,io,data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: portalocker>=2.8.0
+Requires-Dist: metaclass-registry
+Requires-Dist: zarr<3.0,>=2.18.0
+Requires-Dist: ome-zarr>=0.11.0
+Provides-Extra: zarr
+Requires-Dist: zarr<3.0,>=2.18.0; extra == "zarr"
+Requires-Dist: ome-zarr>=0.11.0; extra == "zarr"
+Provides-Extra: torch
+Requires-Dist: torch>=2.0.0; extra == "torch"
+Provides-Extra: jax
+Requires-Dist: jax>=0.4.0; extra == "jax"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow>=2.12.0; extra == "tensorflow"
+Provides-Extra: cupy
+Requires-Dist: cupy>=12.0.0; extra == "cupy"
+Provides-Extra: streaming
+Requires-Dist: pyzmq>=25.0.0; extra == "streaming"
+Provides-Extra: all
+Requires-Dist: zarr<3.0,>=2.18.0; extra == "all"
+Requires-Dist: ome-zarr>=0.11.0; extra == "all"
+Requires-Dist: torch>=2.0.0; extra == "all"
+Requires-Dist: jax>=0.4.0; extra == "all"
+Requires-Dist: tensorflow>=2.12.0; extra == "all"
+Requires-Dist: cupy>=12.0.0; extra == "all"
+Requires-Dist: pyzmq>=25.0.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: coverage>=7.3.2; extra == "dev"
+Requires-Dist: genbadge[coverage]; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=4.0.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
+Requires-Dist: sphinx-toolbox>=3.0.0; extra == "docs"
+Requires-Dist: sphinx-design>=0.5.0; extra == "docs"
+Provides-Extra: gpu
+Requires-Dist: zarr<3.0,>=2.18.0; extra == "gpu"
+Requires-Dist: ome-zarr>=0.11.0; extra == "gpu"
+Requires-Dist: torch<2.8.0,>=2.6.0; extra == "gpu"
+Requires-Dist: torchvision<0.23.0,>=0.21.0; extra == "gpu"
+Requires-Dist: jax<0.6.0,>=0.5.3; extra == "gpu"
+Requires-Dist: jaxlib<0.6.0,>=0.5.3; extra == "gpu"
+Requires-Dist: jax-cuda12-pjrt<0.6.0,>=0.5.3; extra == "gpu"
+Requires-Dist: jax-cuda12-plugin<0.6.0,>=0.5.3; extra == "gpu"
+Requires-Dist: cupy-cuda12x<14.0.0,>=13.3.0; extra == "gpu"
+Requires-Dist: cucim-cu12<26.0.0,>=25.6.0; extra == "gpu"
+Requires-Dist: tensorflow<2.20.0,>=2.19.0; extra == "gpu"
+Requires-Dist: tensorflow-probability[tf]<0.26.0,>=0.25.0; extra == "gpu"
+Requires-Dist: pyclesperanto>=0.17.1; extra == "gpu"
+Dynamic: license-file
+
+# polystore
+
+**Framework-agnostic multi-backend storage abstraction for ML and scientific computing**
+
+[![PyPI version](https://badge.fury.io/py/polystore.svg)](https://badge.fury.io/py/polystore)
+[![Documentation Status](https://readthedocs.org/projects/polystore/badge/?version=latest)](https://polystore.readthedocs.io/en/latest/?badge=latest)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Coverage](https://raw.githubusercontent.com/trissim/polystore/main/.github/badges/coverage.svg)](https://trissim.github.io/polystore/coverage/)
+
+## Features
+
+- **Pluggable Backends**: Disk, memory, Zarr, and streaming backends with auto-registration
+- **Multi-Framework I/O**: Seamless support for NumPy, PyTorch, JAX, TensorFlow, CuPy
+- **Atomic Operations**: Cross-platform atomic file writes with automatic locking
+- **Batch Operations**: Efficient batch loading and saving
+- **Format Detection**: Automatic format detection and routing
+- **Type-Safe**: Full type hints and mypy support
+- **Zero Dependencies**: Core requires only NumPy (framework support is optional)
+
+## Quick Start
+
+```python
+from polystore import FileManager, BackendRegistry
+
+# Create registry and file manager
+registry = BackendRegistry()
+fm = FileManager(registry)
+
+# Save data to disk
+import numpy as np
+data = np.array([[1, 2], [3, 4]])
+fm.save(data, "output.npy", backend="disk")
+
+# Load data back
+loaded = fm.load("output.npy", backend="disk")
+
+# Use memory backend for testing
+fm.save(data, "test.npy", backend="memory")
+cached = fm.load("test.npy", backend="memory")
+```
+
+## Installation
+
+```bash
+# Base installation (NumPy only)
+pip install polystore
+
+# With specific frameworks
+pip install polystore[zarr]
+pip install polystore[torch]
+pip install polystore[jax]
+pip install polystore[tensorflow]
+pip install polystore[cupy]
+
+# With streaming support
+pip install polystore[streaming]
+
+# With all optional dependencies
+pip install polystore[all]
+```
+
+## Supported Backends
+
+| Backend | Description | Storage | Dependencies |
+|---------|-------------|---------|--------------|
+| **disk** | Local filesystem | Persistent | None |
+| **memory** | In-memory cache | Volatile | None |
+| **zarr** | Zarr/OME-Zarr arrays | Persistent | zarr, ome-zarr |
+| **streaming** | ZeroMQ streaming | None | pyzmq |
+
+## Supported Formats
+
+| Format | Extensions | Frameworks |
+|--------|-----------|------------|
+| **NumPy** | `.npy`, `.npz` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **TIFF** | `.tif`, `.tiff` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **Zarr** | `.zarr` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **PyTorch** | `.pt`, `.pth` | PyTorch |
+| **CSV** | `.csv` | NumPy, pandas |
+| **JSON** | `.json` | Python dicts |
+
+## Architecture
+
+```
+polystore/
+├── base.py              # Abstract interfaces (DataSink, DataSource, StorageBackend)
+├── backend_registry.py  # Auto-registration system
+├── disk.py              # Disk storage backend
+├── memory.py            # In-memory backend
+├── zarr.py              # Zarr backend
+├── streaming.py         # ZeroMQ streaming backend
+├── filemanager.py       # High-level API
+├── atomic.py            # Atomic file operations
+└── exceptions.py        # Custom exceptions
+```
+
+## Advanced Usage
+
+### Custom Backends
+
+```python
+from polystore import StorageBackend
+
+class MyBackend(StorageBackend):
+    _backend_type = 'my_backend'  # Auto-registers
+    
+    def save(self, data, file_path, **kwargs):
+        # Your save logic
+        pass
+    
+    def load(self, file_path, **kwargs):
+        # Your load logic
+        pass
+```
+
+### Batch Operations
+
+```python
+# Save multiple files
+data_list = [np.random.rand(100, 100) for _ in range(10)]
+paths = [f"image_{i}.npy" for i in range(10)]
+fm.save_batch(data_list, paths, backend="disk")
+
+# Load multiple files
+loaded_list = fm.load_batch(paths, backend="disk")
+```
+
+### Atomic Writes
+
+```python
+from polystore import atomic_write, atomic_write_json
+
+# Atomic file write with automatic locking
+with atomic_write("output.txt") as f:
+    f.write("data")
+
+# Atomic JSON write
+atomic_write_json({"key": "value"}, "config.json")
+```
+
+## Why polystore?
+
+**Before** (Manual backend management):
+```python
+if backend == 'disk':
+    np.save(path, data)
+elif backend == 'memory':
+    cache[path] = data
+elif backend == 'zarr':
+    zarr.save(path, data)
+# ... 50 more lines of if/elif ...
+```
+
+**After** (polystore):
+```python
+fm.save(data, path, backend=backend)
+```
+
+## Documentation
+
+Full documentation available at [polystore.readthedocs.io](https://polystore.readthedocs.io)
+
+## Addons
+
+Extend polystore with additional backends:
+
+- **polystore-napari**: Napari viewer streaming backend
+- **polystore-fiji**: Fiji/ImageJ streaming backend
+- **polystore-omero**: OMERO server backend
+
+## Performance
+
+- **Zero-copy** conversions between frameworks via DLPack (when possible)
+- **Lazy loading** for optional dependencies
+- **Batch operations** for efficient I/O
+- **Atomic writes** with minimal overhead
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions welcome! Please see CONTRIBUTING.md for guidelines.
+
+## Credits
+
+Developed by Tristan Simas.

polystore-0.1.0/README.md

@@ -0,0 +1,189 @@
+# polystore
+
+**Framework-agnostic multi-backend storage abstraction for ML and scientific computing**
+
+[![PyPI version](https://badge.fury.io/py/polystore.svg)](https://badge.fury.io/py/polystore)
+[![Documentation Status](https://readthedocs.org/projects/polystore/badge/?version=latest)](https://polystore.readthedocs.io/en/latest/?badge=latest)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Coverage](https://raw.githubusercontent.com/trissim/polystore/main/.github/badges/coverage.svg)](https://trissim.github.io/polystore/coverage/)
+
+## Features
+
+- **Pluggable Backends**: Disk, memory, Zarr, and streaming backends with auto-registration
+- **Multi-Framework I/O**: Seamless support for NumPy, PyTorch, JAX, TensorFlow, CuPy
+- **Atomic Operations**: Cross-platform atomic file writes with automatic locking
+- **Batch Operations**: Efficient batch loading and saving
+- **Format Detection**: Automatic format detection and routing
+- **Type-Safe**: Full type hints and mypy support
+- **Zero Dependencies**: Core requires only NumPy (framework support is optional)
+
+## Quick Start
+
+```python
+from polystore import FileManager, BackendRegistry
+
+# Create registry and file manager
+registry = BackendRegistry()
+fm = FileManager(registry)
+
+# Save data to disk
+import numpy as np
+data = np.array([[1, 2], [3, 4]])
+fm.save(data, "output.npy", backend="disk")
+
+# Load data back
+loaded = fm.load("output.npy", backend="disk")
+
+# Use memory backend for testing
+fm.save(data, "test.npy", backend="memory")
+cached = fm.load("test.npy", backend="memory")
+```
+
+## Installation
+
+```bash
+# Base installation (NumPy only)
+pip install polystore
+
+# With specific frameworks
+pip install polystore[zarr]
+pip install polystore[torch]
+pip install polystore[jax]
+pip install polystore[tensorflow]
+pip install polystore[cupy]
+
+# With streaming support
+pip install polystore[streaming]
+
+# With all optional dependencies
+pip install polystore[all]
+```
+
+## Supported Backends
+
+| Backend | Description | Storage | Dependencies |
+|---------|-------------|---------|--------------|
+| **disk** | Local filesystem | Persistent | None |
+| **memory** | In-memory cache | Volatile | None |
+| **zarr** | Zarr/OME-Zarr arrays | Persistent | zarr, ome-zarr |
+| **streaming** | ZeroMQ streaming | None | pyzmq |
+
+## Supported Formats
+
+| Format | Extensions | Frameworks |
+|--------|-----------|------------|
+| **NumPy** | `.npy`, `.npz` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **TIFF** | `.tif`, `.tiff` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **Zarr** | `.zarr` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **PyTorch** | `.pt`, `.pth` | PyTorch |
+| **CSV** | `.csv` | NumPy, pandas |
+| **JSON** | `.json` | Python dicts |
+
+## Architecture
+
+```
+polystore/
+├── base.py              # Abstract interfaces (DataSink, DataSource, StorageBackend)
+├── backend_registry.py  # Auto-registration system
+├── disk.py              # Disk storage backend
+├── memory.py            # In-memory backend
+├── zarr.py              # Zarr backend
+├── streaming.py         # ZeroMQ streaming backend
+├── filemanager.py       # High-level API
+├── atomic.py            # Atomic file operations
+└── exceptions.py        # Custom exceptions
+```
+
+## Advanced Usage
+
+### Custom Backends
+
+```python
+from polystore import StorageBackend
+
+class MyBackend(StorageBackend):
+    _backend_type = 'my_backend'  # Auto-registers
+    
+    def save(self, data, file_path, **kwargs):
+        # Your save logic
+        pass
+    
+    def load(self, file_path, **kwargs):
+        # Your load logic
+        pass
+```
+
+### Batch Operations
+
+```python
+# Save multiple files
+data_list = [np.random.rand(100, 100) for _ in range(10)]
+paths = [f"image_{i}.npy" for i in range(10)]
+fm.save_batch(data_list, paths, backend="disk")
+
+# Load multiple files
+loaded_list = fm.load_batch(paths, backend="disk")
+```
+
+### Atomic Writes
+
+```python
+from polystore import atomic_write, atomic_write_json
+
+# Atomic file write with automatic locking
+with atomic_write("output.txt") as f:
+    f.write("data")
+
+# Atomic JSON write
+atomic_write_json({"key": "value"}, "config.json")
+```
+
+## Why polystore?
+
+**Before** (Manual backend management):
+```python
+if backend == 'disk':
+    np.save(path, data)
+elif backend == 'memory':
+    cache[path] = data
+elif backend == 'zarr':
+    zarr.save(path, data)
+# ... 50 more lines of if/elif ...
+```
+
+**After** (polystore):
+```python
+fm.save(data, path, backend=backend)
+```
+
+## Documentation
+
+Full documentation available at [polystore.readthedocs.io](https://polystore.readthedocs.io)
+
+## Addons
+
+Extend polystore with additional backends:
+
+- **polystore-napari**: Napari viewer streaming backend
+- **polystore-fiji**: Fiji/ImageJ streaming backend
+- **polystore-omero**: OMERO server backend
+
+## Performance
+
+- **Zero-copy** conversions between frameworks via DLPack (when possible)
+- **Lazy loading** for optional dependencies
+- **Batch operations** for efficient I/O
+- **Atomic writes** with minimal overhead
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions welcome! Please see CONTRIBUTING.md for guidelines.
+
+## Credits
+
+Developed by Tristan Simas.

polystore-0.1.0/pyproject.toml

@@ -0,0 +1,200 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "polystore"
+version = "0.1.0"
+description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT License"}
+authors = [
+    {name = "Tristan Simas", email = "tristan.simas@mail.mcgill.ca"}
+]
+keywords = [
+    "storage",
+    "backend",
+    "multi-framework",
+    "numpy",
+    "pytorch",
+    "jax",
+    "tensorflow",
+    "zarr",
+    "io",
+    "data",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    "numpy>=1.26.0",
+    "portalocker>=2.8.0",  # Cross-platform file locking
+    "metaclass-registry",
+    "zarr>=2.18.0,<3.0",  # Required for ZarrStorageBackend
+    "ome-zarr>=0.11.0",   # Required for OME-ZARR HCS compliance
+]
+
+[project.optional-dependencies]
+zarr = [
+    "zarr>=2.18.0,<3.0",
+    "ome-zarr>=0.11.0",
+]
+
+torch = [
+    "torch>=2.0.0",
+]
+
+jax = [
+    "jax>=0.4.0",
+]
+
+tensorflow = [
+    "tensorflow>=2.12.0",
+]
+
+cupy = [
+    "cupy>=12.0.0",
+]
+
+streaming = [
+    "pyzmq>=25.0.0",
+]
+
+all = [
+    "zarr>=2.18.0,<3.0",
+    "ome-zarr>=0.11.0",
+    "torch>=2.0.0",
+    "jax>=0.4.0",
+    "tensorflow>=2.12.0",
+    "cupy>=12.0.0",
+    "pyzmq>=25.0.0",
+]
+
+dev = [
+    "pytest>=7.4.0",
+    "pytest-cov>=4.1.0",
+    "coverage>=7.3.2",
+    "genbadge[coverage]",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+]
+
+docs = [
+    "sphinx>=4.0.0",
+    "sphinx-rtd-theme>=1.0.0",
+    "sphinx-toolbox>=3.0.0",
+    "sphinx-design>=0.5.0",
+]
+
+gpu = [
+    # Zarr
+    "zarr>=2.18.0,<3.0",
+    "ome-zarr>=0.11.0",
+
+    # PyTorch
+    "torch>=2.6.0,<2.8.0",
+    "torchvision>=0.21.0,<0.23.0",
+
+    # JAX
+    "jax>=0.5.3,<0.6.0",
+    "jaxlib>=0.5.3,<0.6.0",
+
+    # JAX CUDA plugins
+    "jax-cuda12-pjrt>=0.5.3,<0.6.0",
+    "jax-cuda12-plugin>=0.5.3,<0.6.0",
+
+    # CuPy
+    "cupy-cuda12x>=13.3.0,<14.0.0",
+
+    # CuCIM
+    "cucim-cu12>=25.6.0,<26.0.0",
+
+    # TensorFlow
+    "tensorflow>=2.19.0,<2.20.0",
+
+    # TensorFlow Probability
+    "tensorflow-probability[tf]>=0.25.0,<0.26.0",
+
+    # pyclesperanto
+    "pyclesperanto>=0.17.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/OpenHCSDev/PolyStore"
+Documentation = "https://polystore.readthedocs.io"
+"Bug Reports" = "https://github.com/OpenHCSDev/PolyStore/issues"
+Source = "https://github.com/OpenHCSDev/PolyStore"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["polystore*"]
+
+
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "--strict-markers",
+    "--strict-config",
+    "--cov=polystore",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--cov-report=xml",
+]
+
+[tool.coverage.run]
+source = ["polystore"]
+omit = [
+    "*/tests/*",
+    "*/test_*.py",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+
+[tool.black]
+line-length = 100
+target-version = ["py311", "py312"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "B",  # flake8-bugbear
+    "C4", # flake8-comprehensions
+    "UP", # pyupgrade
+]
+ignore = [
+    "E501",  # line too long (handled by black)
+    "B008",  # do not perform function calls in argument defaults
+    "C901",  # too complex
+]
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["F401"]  # unused imports

polystore-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+