polystore 0.1.2__tar.gz

polystore-0.1.2/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2025 Tristan Simas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

polystore-0.1.2/PKG-INFO

@@ -0,0 +1,249 @@
+Metadata-Version: 2.4
+Name: polystore
+Version: 0.1.2
+Summary: Framework-agnostic multi-backend storage abstraction for ML and scientific computing
+Author-email: Tristan Simas <tristan.simas@mail.mcgill.ca>
+License: MIT License
+Project-URL: Homepage, https://github.com/trissim/polystore
+Project-URL: Documentation, https://polystore.readthedocs.io
+Project-URL: Bug Reports, https://github.com/trissim/polystore/issues
+Project-URL: Source, https://github.com/trissim/polystore
+Keywords: storage,backend,multi-framework,numpy,pytorch,jax,tensorflow,zarr,io,data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: portalocker>=2.8.0
+Requires-Dist: metaclass-registry>=0.1.0
+Provides-Extra: zarr
+Requires-Dist: zarr<3.0,>=2.18.0; extra == "zarr"
+Requires-Dist: ome-zarr>=0.11.0; extra == "zarr"
+Provides-Extra: torch
+Requires-Dist: torch>=2.0.0; extra == "torch"
+Provides-Extra: jax
+Requires-Dist: jax>=0.4.0; extra == "jax"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow>=2.12.0; extra == "tensorflow"
+Provides-Extra: cupy
+Requires-Dist: cupy>=12.0.0; extra == "cupy"
+Provides-Extra: streaming
+Requires-Dist: pyzmq>=25.0.0; extra == "streaming"
+Provides-Extra: all
+Requires-Dist: zarr<3.0,>=2.18.0; extra == "all"
+Requires-Dist: ome-zarr>=0.11.0; extra == "all"
+Requires-Dist: torch>=2.0.0; extra == "all"
+Requires-Dist: jax>=0.4.0; extra == "all"
+Requires-Dist: tensorflow>=2.12.0; extra == "all"
+Requires-Dist: cupy>=12.0.0; extra == "all"
+Requires-Dist: pyzmq>=25.0.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: coverage>=7.3.2; extra == "dev"
+Requires-Dist: genbadge[coverage]; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=4.0.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
+Requires-Dist: sphinx-toolbox>=3.0.0; extra == "docs"
+Requires-Dist: sphinx-design>=0.5.0; extra == "docs"
+Dynamic: license-file
+
+# polystore
+
+**Framework-agnostic multi-backend storage abstraction for ML and scientific computing**
+
+[![PyPI version](https://badge.fury.io/py/polystore.svg)](https://badge.fury.io/py/polystore)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- **Pluggable Backends**: Disk, memory, Zarr, and streaming backends with auto-registration
+- **Multi-Framework I/O**: Seamless support for NumPy, PyTorch, JAX, TensorFlow, CuPy
+- **Atomic Operations**: Cross-platform atomic file writes with automatic locking
+- **Batch Operations**: Efficient batch loading and saving
+- **Format Detection**: Automatic format detection and routing
+- **Type-Safe**: Full type hints and mypy support
+- **Zero Dependencies**: Core requires only NumPy (framework support is optional)
+
+## Quick Start
+
+```python
+from polystore import FileManager, BackendRegistry
+
+# Create registry and file manager
+registry = BackendRegistry()
+fm = FileManager(registry)
+
+# Save data to disk
+import numpy as np
+data = np.array([[1, 2], [3, 4]])
+fm.save(data, "output.npy", backend="disk")
+
+# Load data back
+loaded = fm.load("output.npy", backend="disk")
+
+# Use memory backend for testing
+fm.save(data, "test.npy", backend="memory")
+cached = fm.load("test.npy", backend="memory")
+```
+
+## Installation
+
+```bash
+# Base installation (NumPy only)
+pip install polystore
+
+# With specific frameworks
+pip install polystore[zarr]
+pip install polystore[torch]
+pip install polystore[jax]
+pip install polystore[tensorflow]
+pip install polystore[cupy]
+
+# With streaming support
+pip install polystore[streaming]
+
+# With all optional dependencies
+pip install polystore[all]
+```
+
+## Supported Backends
+
+| Backend | Description | Storage | Dependencies |
+|---------|-------------|---------|--------------|
+| **disk** | Local filesystem | Persistent | None |
+| **memory** | In-memory cache | Volatile | None |
+| **zarr** | Zarr/OME-Zarr arrays | Persistent | zarr, ome-zarr |
+| **streaming** | ZeroMQ streaming | None | pyzmq |
+
+## Supported Formats
+
+| Format | Extensions | Frameworks |
+|--------|-----------|------------|
+| **NumPy** | `.npy`, `.npz` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **TIFF** | `.tif`, `.tiff` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **Zarr** | `.zarr` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **PyTorch** | `.pt`, `.pth` | PyTorch |
+| **CSV** | `.csv` | NumPy, pandas |
+| **JSON** | `.json` | Python dicts |
+
+## Architecture
+
+```
+polystore/
+├── base.py              # Abstract interfaces (DataSink, DataSource, StorageBackend)
+├── backend_registry.py  # Auto-registration system
+├── disk.py              # Disk storage backend
+├── memory.py            # In-memory backend
+├── zarr.py              # Zarr backend
+├── streaming.py         # ZeroMQ streaming backend
+├── filemanager.py       # High-level API
+├── atomic.py            # Atomic file operations
+└── exceptions.py        # Custom exceptions
+```
+
+## Advanced Usage
+
+### Custom Backends
+
+```python
+from polystore import StorageBackend
+
+class MyBackend(StorageBackend):
+    _backend_type = 'my_backend'  # Auto-registers
+    
+    def save(self, data, file_path, **kwargs):
+        # Your save logic
+        pass
+    
+    def load(self, file_path, **kwargs):
+        # Your load logic
+        pass
+```
+
+### Batch Operations
+
+```python
+# Save multiple files
+data_list = [np.random.rand(100, 100) for _ in range(10)]
+paths = [f"image_{i}.npy" for i in range(10)]
+fm.save_batch(data_list, paths, backend="disk")
+
+# Load multiple files
+loaded_list = fm.load_batch(paths, backend="disk")
+```
+
+### Atomic Writes
+
+```python
+from polystore import atomic_write, atomic_write_json
+
+# Atomic file write with automatic locking
+with atomic_write("output.txt") as f:
+    f.write("data")
+
+# Atomic JSON write
+atomic_write_json({"key": "value"}, "config.json")
+```
+
+## Why polystore?
+
+**Before** (Manual backend management):
+```python
+if backend == 'disk':
+    np.save(path, data)
+elif backend == 'memory':
+    cache[path] = data
+elif backend == 'zarr':
+    zarr.save(path, data)
+# ... 50 more lines of if/elif ...
+```
+
+**After** (polystore):
+```python
+fm.save(data, path, backend=backend)
+```
+
+## Documentation
+
+Full documentation available at [polystore.readthedocs.io](https://polystore.readthedocs.io)
+
+## Addons
+
+Extend polystore with additional backends:
+
+- **polystore-napari**: Napari viewer streaming backend
+- **polystore-fiji**: Fiji/ImageJ streaming backend
+- **polystore-omero**: OMERO server backend
+
+## Performance
+
+- **Zero-copy** conversions between frameworks via DLPack (when possible)
+- **Lazy loading** for optional dependencies
+- **Batch operations** for efficient I/O
+- **Atomic writes** with minimal overhead
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions welcome! Please see CONTRIBUTING.md for guidelines.
+
+## Credits
+
+Developed by Tristan Simas. Extracted from the [OpenHCS](https://github.com/trissim/openhcs) project.
+

polystore-0.1.2/README.md

@@ -0,0 +1,188 @@
+# polystore
+
+**Framework-agnostic multi-backend storage abstraction for ML and scientific computing**
+
+[![PyPI version](https://badge.fury.io/py/polystore.svg)](https://badge.fury.io/py/polystore)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- **Pluggable Backends**: Disk, memory, Zarr, and streaming backends with auto-registration
+- **Multi-Framework I/O**: Seamless support for NumPy, PyTorch, JAX, TensorFlow, CuPy
+- **Atomic Operations**: Cross-platform atomic file writes with automatic locking
+- **Batch Operations**: Efficient batch loading and saving
+- **Format Detection**: Automatic format detection and routing
+- **Type-Safe**: Full type hints and mypy support
+- **Zero Dependencies**: Core requires only NumPy (framework support is optional)
+
+## Quick Start
+
+```python
+from polystore import FileManager, BackendRegistry
+
+# Create registry and file manager
+registry = BackendRegistry()
+fm = FileManager(registry)
+
+# Save data to disk
+import numpy as np
+data = np.array([[1, 2], [3, 4]])
+fm.save(data, "output.npy", backend="disk")
+
+# Load data back
+loaded = fm.load("output.npy", backend="disk")
+
+# Use memory backend for testing
+fm.save(data, "test.npy", backend="memory")
+cached = fm.load("test.npy", backend="memory")
+```
+
+## Installation
+
+```bash
+# Base installation (NumPy only)
+pip install polystore
+
+# With specific frameworks
+pip install polystore[zarr]
+pip install polystore[torch]
+pip install polystore[jax]
+pip install polystore[tensorflow]
+pip install polystore[cupy]
+
+# With streaming support
+pip install polystore[streaming]
+
+# With all optional dependencies
+pip install polystore[all]
+```
+
+## Supported Backends
+
+| Backend | Description | Storage | Dependencies |
+|---------|-------------|---------|--------------|
+| **disk** | Local filesystem | Persistent | None |
+| **memory** | In-memory cache | Volatile | None |
+| **zarr** | Zarr/OME-Zarr arrays | Persistent | zarr, ome-zarr |
+| **streaming** | ZeroMQ streaming | None | pyzmq |
+
+## Supported Formats
+
+| Format | Extensions | Frameworks |
+|--------|-----------|------------|
+| **NumPy** | `.npy`, `.npz` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **TIFF** | `.tif`, `.tiff` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **Zarr** | `.zarr` | NumPy, PyTorch, JAX, TensorFlow, CuPy |
+| **PyTorch** | `.pt`, `.pth` | PyTorch |
+| **CSV** | `.csv` | NumPy, pandas |
+| **JSON** | `.json` | Python dicts |
+
+## Architecture
+
+```
+polystore/
+├── base.py              # Abstract interfaces (DataSink, DataSource, StorageBackend)
+├── backend_registry.py  # Auto-registration system
+├── disk.py              # Disk storage backend
+├── memory.py            # In-memory backend
+├── zarr.py              # Zarr backend
+├── streaming.py         # ZeroMQ streaming backend
+├── filemanager.py       # High-level API
+├── atomic.py            # Atomic file operations
+└── exceptions.py        # Custom exceptions
+```
+
+## Advanced Usage
+
+### Custom Backends
+
+```python
+from polystore import StorageBackend
+
+class MyBackend(StorageBackend):
+    _backend_type = 'my_backend'  # Auto-registers
+    
+    def save(self, data, file_path, **kwargs):
+        # Your save logic
+        pass
+    
+    def load(self, file_path, **kwargs):
+        # Your load logic
+        pass
+```
+
+### Batch Operations
+
+```python
+# Save multiple files
+data_list = [np.random.rand(100, 100) for _ in range(10)]
+paths = [f"image_{i}.npy" for i in range(10)]
+fm.save_batch(data_list, paths, backend="disk")
+
+# Load multiple files
+loaded_list = fm.load_batch(paths, backend="disk")
+```
+
+### Atomic Writes
+
+```python
+from polystore import atomic_write, atomic_write_json
+
+# Atomic file write with automatic locking
+with atomic_write("output.txt") as f:
+    f.write("data")
+
+# Atomic JSON write
+atomic_write_json({"key": "value"}, "config.json")
+```
+
+## Why polystore?
+
+**Before** (Manual backend management):
+```python
+if backend == 'disk':
+    np.save(path, data)
+elif backend == 'memory':
+    cache[path] = data
+elif backend == 'zarr':
+    zarr.save(path, data)
+# ... 50 more lines of if/elif ...
+```
+
+**After** (polystore):
+```python
+fm.save(data, path, backend=backend)
+```
+
+## Documentation
+
+Full documentation available at [polystore.readthedocs.io](https://polystore.readthedocs.io)
+
+## Addons
+
+Extend polystore with additional backends:
+
+- **polystore-napari**: Napari viewer streaming backend
+- **polystore-fiji**: Fiji/ImageJ streaming backend
+- **polystore-omero**: OMERO server backend
+
+## Performance
+
+- **Zero-copy** conversions between frameworks via DLPack (when possible)
+- **Lazy loading** for optional dependencies
+- **Batch operations** for efficient I/O
+- **Atomic writes** with minimal overhead
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions welcome! Please see CONTRIBUTING.md for guidelines.
+
+## Credits
+
+Developed by Tristan Simas. Extracted from the [OpenHCS](https://github.com/trissim/openhcs) project.
+

polystore-0.1.2/pyproject.toml

@@ -0,0 +1,167 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "polystore"
+dynamic = ["version"]
+description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT License"}
+authors = [
+    {name = "Tristan Simas", email = "tristan.simas@mail.mcgill.ca"}
+]
+keywords = [
+    "storage",
+    "backend",
+    "multi-framework",
+    "numpy",
+    "pytorch",
+    "jax",
+    "tensorflow",
+    "zarr",
+    "io",
+    "data",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    "numpy>=1.26.0",
+    "portalocker>=2.8.0",  # Cross-platform file locking
+    "metaclass-registry>=0.1.0",
+]
+
+[project.optional-dependencies]
+zarr = [
+    "zarr>=2.18.0,<3.0",
+    "ome-zarr>=0.11.0",
+]
+
+torch = [
+    "torch>=2.0.0",
+]
+
+jax = [
+    "jax>=0.4.0",
+]
+
+tensorflow = [
+    "tensorflow>=2.12.0",
+]
+
+cupy = [
+    "cupy>=12.0.0",
+]
+
+streaming = [
+    "pyzmq>=25.0.0",
+]
+
+all = [
+    "zarr>=2.18.0,<3.0",
+    "ome-zarr>=0.11.0",
+    "torch>=2.0.0",
+    "jax>=0.4.0",
+    "tensorflow>=2.12.0",
+    "cupy>=12.0.0",
+    "pyzmq>=25.0.0",
+]
+
+dev = [
+    "pytest>=7.4.0",
+    "pytest-cov>=4.1.0",
+    "coverage>=7.3.2",
+    "genbadge[coverage]",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+]
+
+docs = [
+    "sphinx>=4.0.0",
+    "sphinx-rtd-theme>=1.0.0",
+    "sphinx-toolbox>=3.0.0",
+    "sphinx-design>=0.5.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/trissim/polystore"
+Documentation = "https://polystore.readthedocs.io"
+"Bug Reports" = "https://github.com/trissim/polystore/issues"
+Source = "https://github.com/trissim/polystore"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["polystore*"]
+
+[tool.setuptools.dynamic]
+version = {attr = "polystore.__version__"}
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "--strict-markers",
+    "--strict-config",
+    "--cov=polystore",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--cov-report=xml",
+]
+
+[tool.coverage.run]
+source = ["polystore"]
+omit = [
+    "*/tests/*",
+    "*/test_*.py",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+
+[tool.black]
+line-length = 100
+target-version = ["py311", "py312"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "I",  # isort
+    "B",  # flake8-bugbear
+    "C4", # flake8-comprehensions
+    "UP", # pyupgrade
+]
+ignore = [
+    "E501",  # line too long (handled by black)
+    "B008",  # do not perform function calls in argument defaults
+    "C901",  # too complex
+]
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["F401"]  # unused imports
+

polystore-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

polystore-0.1.2/src/polystore/__init__.py

@@ -0,0 +1,80 @@
+"""
+Polystore - Framework-agnostic multi-backend storage abstraction.
+
+Provides pluggable storage backends with multi-framework I/O support for
+NumPy, PyTorch, JAX, TensorFlow, CuPy, and Zarr.
+"""
+
+__version__ = "0.1.2"
+
+# Core abstractions
+from .base import (
+    DataSink,
+    DataSource,
+    StorageBackend,
+    VirtualBackend,
+    ReadOnlyBackend,
+)
+
+# Concrete backends
+from .memory import MemoryBackend
+from .disk import DiskBackend
+
+# Optional backends
+try:
+    from .zarr import ZarrBackend
+except ImportError:
+    ZarrBackend = None
+
+# File manager
+from .filemanager import FileManager
+
+# Registry
+from .backend_registry import BackendRegistry, create_storage_registry
+
+# Atomic operations
+from .atomic import atomic_write, atomic_write_json
+
+# Exceptions
+from .exceptions import (
+    StorageError,
+    StorageResolutionError,
+    BackendNotFoundError,
+    UnsupportedFormatError,
+)
+
+# Streaming (optional)
+try:
+    from .streaming import StreamingBackend
+except ImportError:
+    StreamingBackend = None
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core abstractions
+    "DataSink",
+    "DataSource",
+    "StorageBackend",
+    "VirtualBackend",
+    "ReadOnlyBackend",
+    # Backends
+    "MemoryBackend",
+    "DiskBackend",
+    "ZarrBackend",
+    # File manager
+    "FileManager",
+    # Registry
+    "BackendRegistry",
+    # Atomic operations
+    "atomic_write",
+    "atomic_write_json",
+    # Exceptions
+    "StorageError",
+    "StorageResolutionError",
+    "BackendNotFoundError",
+    "UnsupportedFormatError",
+    # Streaming
+    "StreamingBackend",
+]
+