cellarr-array 0.0.1__tar.gz → 0.0.3__tar.gz

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/.github/workflows/publish-pypi.yml

@@ -52,4 +52,4 @@ jobs:
 
       # This uses the trusted publisher workflow so no token is required.
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.8.2
+  rev: v0.11.5
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/CHANGELOG.md

@@ -1,13 +1,17 @@
 # Changelog
 
+## Version 0.0.2
+
+- Support in-memory tiledb objects. Updated tests and documentation.
+
 ## Version 0.0.1
 
-Initial implementation of the sparse and dense arrays backed by TileDB. 
+Initial implementation of the sparse and dense arrays backed by TileDB.
 
 - Supports reading of objects
-  - Directly slices the TileDB object is all arguments to subset are contiguous blocks. 
-  - Otherwise redirects them to `multi_index`, if one of the argument to subset is a slice, drops the last because of inclusive upper bounds in this method. 
+  - Directly slices the TileDB object is all arguments to subset are contiguous blocks.
+  - Otherwise redirects them to `multi_index`, if one of the argument to subset is a slice, drops the last because of inclusive upper bounds in this method.
 
   This helps keeps slicing consistent across various operations and trying to be performant in the process.
 
-- Supports writing of various data objects into dense and sparse arrays. Expects all chunks to be aligned along the rows. 
+- Supports writing of various data objects into dense and sparse arrays. Expects all chunks to be aligned along the rows.

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: cellarr-array
-Version: 0.0.1
+Version: 0.0.3
 Summary: Base class for handling TileDB backed arrays.
 Home-page: https://github.com/cellarr/cellarr-array
 Author: Jayaram Kancherla
@@ -20,13 +20,14 @@ Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"
+Dynamic: license-file
 
 [![PyPI-Server](https://img.shields.io/pypi/v/cellarr-array.svg)](https://pypi.org/project/cellarr-array/)
 ![Unit tests](https://github.com/cellarr/cellarr-array/actions/workflows/run-tests.yml/badge.svg)
 
 # cellarr-array
 
-This package provided high-level wrappers for TileDB arrays optimized for handling genomic data matrices.
+This package provided high-level wrappers for TileDB arrays, for handling genomic data matrices.
 
 ## Install
 
@@ -116,24 +117,43 @@ subset = dense_array[100:200, genes]
 ### Working with Sparse Arrays
 
 ```python
-# Create a sparse array with COO output format
-coo_array = SparseCellArray(
+from cellarr_array import SparseCellArray
+
+# Create a sparse array with CSR output format
+csr_array = SparseCellArray(
     uri="sparse_matrix.tdb",
-    return_coo=True
+    return_sparse=True
 )
 
-# Get result as COO matrix
-result = coo_array[100:200, 500:1000]
+# Get result as CSR matrix
+result = csr_array[100:200, 500:1000]
 
 # Result is scipy.sparse.coo_matrix
-assert sparse.isspmatrix_coo(result)
+assert sparse.isspmatrix_csr(result)
 
 # Perform sparse operations
 nnz = result.nnz
 density = result.nnz / (result.shape[0] * result.shape[1])
 
 # Convert to other sparse formats if needed
-result_csr = result.tocsr()
+result_csc = result.tocsc()
+```
+
+Likewise create a CSC output format
+
+```python
+from scipy import sparse
+
+# Create a sparse array with CSC output format
+csc_array = SparseCellArray(
+    uri="sparse_matrix.tdb",
+    return_sparse=True,
+    sparse_coerce=sparse.csc_matrix
+)
+
+# Get result as CSR matrix
+result = csc_array[100:200, 500:1000]
+print(result)
 ```
 
 ### Array Maintenance
@@ -144,7 +164,7 @@ array.consolidate()
 
 # Custom consolidation
 config = ConsolidationConfig(
-    steps=["fragment"],
+    steps=2,
     vacuum_after=True
 )
 array.consolidate(config)

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/README.md

@@ -3,7 +3,7 @@
 
 # cellarr-array
 
-This package provided high-level wrappers for TileDB arrays optimized for handling genomic data matrices.
+This package provided high-level wrappers for TileDB arrays, for handling genomic data matrices.
 
 ## Install
 
@@ -93,24 +93,43 @@ subset = dense_array[100:200, genes]
 ### Working with Sparse Arrays
 
 ```python
-# Create a sparse array with COO output format
-coo_array = SparseCellArray(
+from cellarr_array import SparseCellArray
+
+# Create a sparse array with CSR output format
+csr_array = SparseCellArray(
     uri="sparse_matrix.tdb",
-    return_coo=True
+    return_sparse=True
 )
 
-# Get result as COO matrix
-result = coo_array[100:200, 500:1000]
+# Get result as CSR matrix
+result = csr_array[100:200, 500:1000]
 
 # Result is scipy.sparse.coo_matrix
-assert sparse.isspmatrix_coo(result)
+assert sparse.isspmatrix_csr(result)
 
 # Perform sparse operations
 nnz = result.nnz
 density = result.nnz / (result.shape[0] * result.shape[1])
 
 # Convert to other sparse formats if needed
-result_csr = result.tocsr()
+result_csc = result.tocsc()
+```
+
+Likewise create a CSC output format
+
+```python
+from scipy import sparse
+
+# Create a sparse array with CSC output format
+csc_array = SparseCellArray(
+    uri="sparse_matrix.tdb",
+    return_sparse=True,
+    sparse_coerce=sparse.csc_matrix
+)
+
+# Get result as CSR matrix
+result = csc_array[100:200, 500:1000]
+print(result)
 ```
 
 ### Array Maintenance
@@ -121,7 +140,7 @@ array.consolidate()
 
 # Custom consolidation
 config = ConsolidationConfig(
-    steps=["fragment"],
+    steps=2,
     vacuum_after=True
 )
 array.consolidate(config)

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/docs/conf.py

@@ -311,7 +311,7 @@ extensions.remove('myst_parser')
 extensions.append('myst_nb')
 
 # Less verbose api documentation
-# extensions.append('sphinx_autodoc_typehints')
+extensions.append('sphinx_autodoc_typehints')
 
 autodoc_default_options = {
     "special-members": True,

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/docs/index.md

@@ -1,17 +1,6 @@
 # cellarr-array
 
-Add a short description here!
-
-
-## Note
-
-> This is the main page of your project's [Sphinx] documentation. It is
-> formatted in [Markdown]. Add additional pages by creating md-files in
-> `docs` or rst-files (formatted in [reStructuredText]) and adding links to
-> them in the `Contents` section below.
->
-> Please check [Sphinx] and [MyST] for more information
-> about how to document your project and how to configure your preferences.
+This package provided high-level wrappers for TileDB arrays optimized for handling genomic data matrices.
 
 
 ## Contents

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/setup.py

@@ -1,10 +1,10 @@
 """
-    Setup file for cellarr-array.
-    Use setup.cfg to configure your project.
+Setup file for cellarr-array.
+Use setup.cfg to configure your project.
 
-    This file was generated with PyScaffold 4.6.
-    PyScaffold helps you to put up the scaffold of your new Python project.
-    Learn more under: https://pyscaffold.org/
+This file was generated with PyScaffold 4.6.
+PyScaffold helps you to put up the scaffold of your new Python project.
+Learn more under: https://pyscaffold.org/
 """
 
 from setuptools import setup

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array/CellArray.py

@@ -1,5 +1,11 @@
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
+
+try:
+    from types import EllipsisType
+except ImportError:
+    # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025
+    EllipsisType = type(...)
 from typing import List, Literal, Optional, Tuple, Union
 
 import numpy as np
@@ -42,7 +48,8 @@ class CellArray(ABC):
                 Defaults to None for automatic mode switching.
 
             config_or_context:
-                Config or context object.
+                Optional config or context object.
+
                 Defaults to None.
 
             validate:
@@ -53,14 +60,15 @@ class CellArray(ABC):
         self._mode = mode
 
         if config_or_context is None:
-            config_or_context = tiledb.Config()
-
-        if isinstance(config_or_context, tiledb.Config):
-            ctx = tiledb.Ctx(config_or_context)
-        elif isinstance(config_or_context, tiledb.Ctx):
-            ctx = config_or_context
+            # config_or_context = tiledb.Config()
+            ctx = None
         else:
-            raise TypeError("'config_or_context' must be either TileDB config or a context object.")
+            if isinstance(config_or_context, tiledb.Config):
+                ctx = tiledb.Ctx(config_or_context)
+            elif isinstance(config_or_context, tiledb.Ctx):
+                ctx = config_or_context
+            else:
+                raise TypeError("'config_or_context' must be either TileDB config or a context object.")
 
         self._ctx = ctx
         self._array = None
@@ -68,6 +76,7 @@ class CellArray(ABC):
         self._ndim = None
         self._dim_names = None
         self._attr_names = None
+        self._nonempty_domain = None
 
         if validate:
             self._validate(attr=attr)
@@ -157,7 +166,7 @@ class CellArray(ABC):
         finally:
             array.close()
 
-    def __getitem__(self, key: Union[slice, Tuple[Union[slice, List[int]], ...]]):
+    def __getitem__(self, key: Union[slice, EllipsisType, Tuple[Union[slice, List[int]], ...], EllipsisType]):
         """Get item implementation that routes to either direct slicing or multi_index
         based on the type of indices provided.
 
@@ -174,16 +183,22 @@ class CellArray(ABC):
         # Normalize all indices
         normalized_key = tuple(SliceHelper.normalize_index(idx, self.shape[i]) for i, idx in enumerate(key))
 
+        num_ellipsis = sum(isinstance(i, EllipsisType) for i in normalized_key)
+        if num_ellipsis > 1:
+            raise IndexError(f"Found more than 1 Ellipsis (...) in key: {normalized_key}")
+
         # Check if we can use direct slicing
-        use_direct = all(isinstance(idx, slice) for idx in normalized_key)
+        use_direct = all(isinstance(idx, (slice, EllipsisType)) for idx in normalized_key)
 
         if use_direct:
             return self._direct_slice(normalized_key)
         else:
+            if num_ellipsis > 0:
+                raise IndexError(f"tiledb does not support ellipsis in multi-index access: {normalized_key}")
             return self._multi_index(normalized_key)
 
     @abstractmethod
-    def _direct_slice(self, key: Tuple[slice, ...]) -> np.ndarray:
+    def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarray:
         """Implementation for direct slicing."""
         pass
 

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array/DenseCellArray.py

@@ -1,3 +1,8 @@
+try:
+    from types import EllipsisType
+except ImportError:
+    # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025
+    EllipsisType = type(...)
 from typing import List, Tuple, Union
 
 import numpy as np
@@ -13,7 +18,7 @@ __license__ = "MIT"
 class DenseCellArray(CellArray):
     """Implementation for dense TileDB arrays."""
 
-    def _direct_slice(self, key: Tuple[slice, ...]) -> np.ndarray:
+    def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarray:
         """Implementation for direct slicing of dense arrays.
 
         Args:

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array/SparseCellArray.py

@@ -1,3 +1,8 @@
+try:
+    from types import EllipsisType
+except ImportError:
+    # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025
+    EllipsisType = type(...)
 from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
@@ -118,7 +123,7 @@ class SparseCellArray(CellArray):
 
         return sliced[key]
 
-    def _direct_slice(self, key: Tuple[slice, ...]) -> Union[np.ndarray, sparse.coo_matrix]:
+    def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> Union[np.ndarray, sparse.coo_matrix]:
         """Implementation for direct slicing of sparse arrays."""
         with self.open_array(mode="r") as array:
             result = array[key]

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array/__init__.py

@@ -18,4 +18,4 @@ finally:
 from .config import CellArrConfig, ConsolidationConfig
 from .DenseCellArray import DenseCellArray
 from .SparseCellArray import SparseCellArray
-from .helpers import create_cellarray, SliceHelper
+from .helpers import create_cellarray, SliceHelper

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array/helpers.py

@@ -1,3 +1,8 @@
+try:
+    from types import EllipsisType
+except ImportError:
+    # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025
+    EllipsisType = type(...)
 from typing import List, Optional, Tuple, Union
 
 import numpy as np
@@ -150,9 +155,12 @@ class SliceHelper:
         return None
 
     @staticmethod
-    def normalize_index(idx: Union[int, slice, List[int]], dim_size: int) -> Union[slice, List[int]]:
+    def normalize_index(idx: Union[int, slice, List[int]], dim_size: int) -> Union[slice, List[int], EllipsisType]:
         """Normalize index to handle negative indices and ensure consistency."""
 
+        if isinstance(idx, EllipsisType):
+            return idx
+
         # Convert ranges to slices
         if isinstance(idx, range):
             idx = slice(idx.start, idx.stop, idx.step)

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: cellarr-array
-Version: 0.0.1
+Version: 0.0.3
 Summary: Base class for handling TileDB backed arrays.
 Home-page: https://github.com/cellarr/cellarr-array
 Author: Jayaram Kancherla
@@ -20,13 +20,14 @@ Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"
+Dynamic: license-file
 
 [![PyPI-Server](https://img.shields.io/pypi/v/cellarr-array.svg)](https://pypi.org/project/cellarr-array/)
 ![Unit tests](https://github.com/cellarr/cellarr-array/actions/workflows/run-tests.yml/badge.svg)
 
 # cellarr-array
 
-This package provided high-level wrappers for TileDB arrays optimized for handling genomic data matrices.
+This package provided high-level wrappers for TileDB arrays, for handling genomic data matrices.
 
 ## Install
 
@@ -116,24 +117,43 @@ subset = dense_array[100:200, genes]
 ### Working with Sparse Arrays
 
 ```python
-# Create a sparse array with COO output format
-coo_array = SparseCellArray(
+from cellarr_array import SparseCellArray
+
+# Create a sparse array with CSR output format
+csr_array = SparseCellArray(
     uri="sparse_matrix.tdb",
-    return_coo=True
+    return_sparse=True
 )
 
-# Get result as COO matrix
-result = coo_array[100:200, 500:1000]
+# Get result as CSR matrix
+result = csr_array[100:200, 500:1000]
 
 # Result is scipy.sparse.coo_matrix
-assert sparse.isspmatrix_coo(result)
+assert sparse.isspmatrix_csr(result)
 
 # Perform sparse operations
 nnz = result.nnz
 density = result.nnz / (result.shape[0] * result.shape[1])
 
 # Convert to other sparse formats if needed
-result_csr = result.tocsr()
+result_csc = result.tocsc()
+```
+
+Likewise create a CSC output format
+
+```python
+from scipy import sparse
+
+# Create a sparse array with CSC output format
+csc_array = SparseCellArray(
+    uri="sparse_matrix.tdb",
+    return_sparse=True,
+    sparse_coerce=sparse.csc_matrix
+)
+
+# Get result as CSR matrix
+result = csc_array[100:200, 500:1000]
+print(result)
 ```
 
 ### Array Maintenance
@@ -144,7 +164,7 @@ array.consolidate()
 
 # Custom consolidation
 config = ConsolidationConfig(
-    steps=["fragment"],
+    steps=2,
     vacuum_after=True
 )
 array.consolidate(config)

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/src/cellarr_array.egg-info/SOURCES.txt

@@ -39,4 +39,5 @@ tests/conftest.py
 tests/test_all.py
 tests/test_dense.py
 tests/test_helpers.py
+tests/test_inmemory.py
 tests/test_sparse.py

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/tests/conftest.py

@@ -16,7 +16,7 @@ from pathlib import Path
 import numpy as np
 import pytest
 
-from cellarr_array import CellArrConfig, create_cellarray
+from cellarr_array import create_cellarray
 
 
 @pytest.fixture(scope="function")

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/tests/test_all.py

@@ -4,7 +4,7 @@ import numpy as np
 import pytest
 from scipy import sparse
 
-from cellarr_array import CellArrConfig, ConsolidationConfig, DenseCellArray, SparseCellArray, create_cellarray
+from cellarr_array import CellArrConfig, ConsolidationConfig, DenseCellArray, create_cellarray
 
 __author__ = "Jayaram Kancherla"
 __copyright__ = "Jayaram Kancherla"
@@ -21,7 +21,7 @@ def test_dimension_validation(temp_dir):
 def test_attribute_validation(temp_dir):
     uri = str(Path(temp_dir) / "attr_test")
 
-    array = create_cellarray(uri=uri, shape=(10, 10), attr_dtype=np.float32, attr_name="values")
+    create_cellarray(uri=uri, shape=(10, 10), attr_dtype=np.float32, attr_name="values")
 
     with pytest.raises(ValueError, match="Attribute 'invalid' does not exist"):
         DenseCellArray(uri, attr="invalid")

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/tests/test_dense.py

@@ -84,6 +84,11 @@ def test_1d_slicing(sample_dense_array_1d):
     result = sample_dense_array_1d[-10:]
     np.testing.assert_array_almost_equal(result, data[-10:])
 
+    # Ellipsis
+    result = sample_dense_array_1d[...]
+    actual = data[...]
+    np.testing.assert_array_almost_equal(result, actual), f"{actual} != {result}"
+
 
 def test_2d_slicing(sample_dense_array_2d):
     data = np.random.random((100, 50)).astype(np.float32)
@@ -105,6 +110,18 @@ def test_2d_slicing(sample_dense_array_2d):
     result = sample_dense_array_2d[-10:, -5:]
     np.testing.assert_array_almost_equal(result, data[-10:, -5:])
 
+    # Ellipsis
+    result = sample_dense_array_2d[..., :1]
+    np.testing.assert_array_almost_equal(result, data[..., :1])
+    result = sample_dense_array_2d[..., :]
+    np.testing.assert_array_almost_equal(result, data[..., :])
+    result = sample_dense_array_2d[-1:, ...]
+    np.testing.assert_array_almost_equal(result, data[-1:, ...])
+    with pytest.raises(IndexError):
+        _ = sample_dense_array_2d[..., ...]
+    with pytest.raises(IndexError):
+        _ = sample_dense_array_2d[[0, 3], ...]
+
 
 def test_multi_index_access(sample_dense_array_2d):
     data = np.random.random((100, 50)).astype(np.float32)

cellarr_array-0.0.3/tests/test_inmemory.py

@@ -0,0 +1,31 @@
+import numpy as np
+import scipy as sp
+
+from cellarr_array import create_cellarray
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def test_inmem_uri():
+    shape = (10_000, 10_000)
+    arr = np.arange(100_000_000).reshape(shape)
+    uri = "mem://dense"
+
+    dense_inmem = create_cellarray(uri, shape=(shape))
+    dense_inmem.write_batch(arr, start_row=0)
+
+    assert np.allclose(dense_inmem[:10, :10], arr[:10, :10])
+
+
+def test_inmem_uri_sparse():
+    shape = (1000, 1000)
+
+    s = sp.sparse.random(1000, 1000, density=0.25)
+    uri = "mem://sparse"
+
+    dense_inmem = create_cellarray(uri, shape=(shape), sparse=True)
+    dense_inmem.write_batch(s, start_row=0)
+
+    assert np.allclose(dense_inmem[:10, :10].toarray(), s.tocsr()[:10, :10].toarray())

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/tests/test_sparse.py

@@ -98,6 +98,11 @@ def test_coo_output(sample_sparse_array_2d):
     assert sparse.isspmatrix_csr(result)
     np.testing.assert_array_almost_equal(result.toarray(), data.toarray())
 
+    # Test full slice with ellipsis
+    result = array_coo[0:10, ...]
+    assert sparse.isspmatrix_csr(result)
+    np.testing.assert_array_almost_equal(result.toarray(), data.toarray())
+
     # Test partial slice
     data_csr = data.tocsr()
     result = array_coo[2:5, 10:20]

{cellarr_array-0.0.1 → cellarr_array-0.0.3}/docs/requirements.txt

@@ -1,9 +1,9 @@
+furo
+myst-nb
 # Requirements file for ReadTheDocs, check .readthedocs.yml.
 # To build the module reference correctly, make sure every external package
 # under `install_requires` in `setup.cfg` is also listed here!
 # sphinx_rtd_theme
 myst-parser[linkify]
 sphinx>=3.2.1
-myst-nb
-furo
 sphinx-autodoc-typehints