cellarr-array 0.1.0__tar.gz → 0.3.0__tar.gz

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/.github/workflows/publish-pypi.yml

@@ -19,10 +19,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Install dependencies
         run: |

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/.github/workflows/run-tests.yml

@@ -28,7 +28,7 @@ jobs:
   test:
     strategy:
       matrix:
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         platform:
           - ubuntu-latest
           # - macos-latest

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/.gitignore

@@ -52,3 +52,5 @@ MANIFEST
 .venv*/
 .conda*/
 .python-version
+
+*.tdb

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.11.5
+  rev: v0.12.1
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## Version 0.3.0
+
+- Support for string dimensions when creating cellarr arrays.
+- Support query conditions for slice operations.
+- Added unique dim values. Only supported for sparse arrays.
+- EOL for Python 3.9
+
+## Version 0.2.0
+
+- Dataloaders for sparse and dense arrays, We provide templates for both map and Iterable style dataloaders. Users are expected the caveats of both of these approaches.
+- Fixed a bug with slicing on 1D arrays and many improvements for optimizing slicing parameters.
+- Update documentation and tests.
+
 ## Version 0.1.0
 
 - Support cellarr-arrays on user provided tiledb array objects.

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: cellarr-array
-Version: 0.1.0
+Version: 0.3.0
 Summary: Base class for handling TileDB backed arrays.
 Home-page: https://github.com/cellarr/cellarr-array
 Author: Jayaram Kancherla
 Author-email: jayaram.kancherla@gmail.com
 License: MIT
 Project-URL: Documentation, https://github.com/cellarr/cellarr-array
+Project-URL: Source, https://github.com/cellarr/cellarr-array
 Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
@@ -16,10 +17,14 @@ Requires-Dist: importlib-metadata; python_version < "3.8"
 Requires-Dist: tiledb
 Requires-Dist: numpy
 Requires-Dist: scipy
+Provides-Extra: optional
+Requires-Dist: torch; extra == "optional"
 Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"
+Requires-Dist: pandas; extra == "testing"
+Requires-Dist: torch; extra == "testing"
 Dynamic: license-file
 
 [![PyPI-Server](https://img.shields.io/pypi/v/cellarr-array.svg)](https://pypi.org/project/cellarr-array/)

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/setup.cfg

@@ -10,6 +10,7 @@ long_description_content_type = text/markdown; charset=UTF-8; variant=GFM
 url = https://github.com/cellarr/cellarr-array
 project_urls = 
 	Documentation = https://github.com/cellarr/cellarr-array
+	Source = https://github.com/cellarr/cellarr-array
 platforms = any
 classifiers = 
 	Development Status :: 4 - Beta
@@ -33,10 +34,14 @@ exclude =
 	tests
 
 [options.extras_require]
+optional = 
+	torch
 testing = 
 	setuptools
 	pytest
 	pytest-cov
+	pandas
+	%(optional)s
 
 [options.entry_points]
 

{cellarr_array-0.1.0 → cellarr_array-0.3.0}/src/cellarr_array/__init__.py

@@ -15,7 +15,6 @@ except PackageNotFoundError:  # pragma: no cover
 finally:
     del version, PackageNotFoundError
 
-from .config import CellArrConfig, ConsolidationConfig
-from .cellarray_dense import DenseCellArray
-from .cellarray_sparse import SparseCellArray
-from .helpers import create_cellarray, SliceHelper
+from .core import DenseCellArray, SparseCellArray
+from .core.helpers import create_cellarray
+from .utils import CellArrConfig, ConsolidationConfig

cellarr_array-0.3.0/src/cellarr_array/core/__init__.py

@@ -0,0 +1,3 @@
+from .base import CellArray
+from .dense import DenseCellArray
+from .sparse import SparseCellArray

cellarr_array-0.1.0/src/cellarr_array/cellarray_base.py → cellarr_array-0.3.0/src/cellarr_array/core/base.py

@@ -12,7 +12,7 @@ import numpy as np
 import tiledb
 from scipy import sparse
 
-from .config import ConsolidationConfig
+from ..utils.config import ConsolidationConfig
 from .helpers import SliceHelper
 
 __author__ = "Jayaram Kancherla"
@@ -69,6 +69,7 @@ class CellArray(ABC):
         self._array_passed_in = False
         self._opened_array_external = None
         self._ctx = None
+        self._dim_dtypes = None
 
         if tiledb_array_obj is not None:
             if not isinstance(tiledb_array_obj, tiledb.Array):
@@ -185,7 +186,16 @@ class CellArray(ABC):
     def shape(self) -> Tuple[int, ...]:
         if self._shape is None:
             with self.open_array(mode="r") as A:
-                self._shape = tuple(int(dim.domain[1] - dim.domain[0] + 1) for dim in A.schema.domain)
+                shape_list = []
+                for dim in A.schema.domain:
+                    try:
+                        # This will fail for string dimensions
+                        shape_list.append(dim.shape[0])
+                    except TypeError:
+                        # For string dimensions, the shape is not well-defined.
+                        # We use a large number as a placeholder for slicing purposes.
+                        shape_list.append(2**63 - 1)
+                self._shape = tuple(shape_list)
         return self._shape
 
     @property
@@ -209,6 +219,14 @@ class CellArray(ABC):
                 # self._ndim = len(self.shape)
         return self._ndim
 
+    @property
+    def dim_dtypes(self) -> List[np.dtype]:
+        """Get dimension dtypes of the array."""
+        if self._dim_dtypes is None:
+            with self.open_array(mode="r") as A:
+                self._dim_dtypes = [dim.dtype for dim in A.schema.domain]
+        return self._dim_dtypes
+
     @contextmanager
     def open_array(self, mode: Optional[str] = None):
         """Context manager for array operations.
@@ -266,15 +284,30 @@ class CellArray(ABC):
         Args:
             key:
                 Slice or list of indices for each dimension in the array.
+
+                Alternatively, may be string to specify query conditions.
         """
+        # This is a query condition
+        if isinstance(key, str):
+            with self.open_array(mode="r") as array:
+                if self._attr is not None:
+                    return array.query(cond=key, attrs=[self._attr])[:]
+                else:
+                    array.query(cond=key)[:]
+
         if not isinstance(key, tuple):
             key = (key,)
 
         if len(key) > self.ndim:
             raise IndexError(f"Invalid number of dimensions: got {len(key)}, expected {self.ndim}")
 
+        if len(key) < self.ndim:
+            key = key + (slice(None),) * (self.ndim - len(key))
+
         # Normalize all indices
-        normalized_key = tuple(SliceHelper.normalize_index(idx, self.shape[i]) for i, idx in enumerate(key))
+        normalized_key = tuple(
+            SliceHelper.normalize_index(idx, self.shape[i], self.dim_dtypes[i]) for i, idx in enumerate(key)
+        )
 
         num_ellipsis = sum(isinstance(i, EllipsisType) for i in normalized_key)
         if num_ellipsis > 1:
@@ -342,3 +375,17 @@ class CellArray(ABC):
                 Additional arguments for write operation.
         """
         pass
+
+    def get_unique_dim_values(self, dim_name: Optional[str] = None) -> np.ndarray:
+        """Get unique values for a dimension.
+
+        Args:
+            dim_name:
+                The name of the dimension. If None, unique values for all
+                dimensions are returned.
+
+        Returns:
+            An array of unique dimension values.
+        """
+        with self.open_array(mode="r") as A:
+            return A.unique_dim_values(dim_name)

cellarr_array-0.1.0/src/cellarr_array/cellarray_dense.py → cellarr_array-0.3.0/src/cellarr_array/core/dense.py

@@ -7,7 +7,7 @@ from typing import List, Tuple, Union
 
 import numpy as np
 
-from .cellarray_base import CellArray
+from .base import CellArray
 from .helpers import SliceHelper
 
 __author__ = "Jayaram Kancherla"
@@ -92,7 +92,6 @@ class DenseCellArray(CellArray):
         if len(data.shape) != self.ndim:
             raise ValueError(f"Data dimensions {data.shape} don't match array dimensions {self.shape}.")
 
-        # Check bounds
         end_row = start_row + data.shape[0]
         if end_row > self.shape[0]:
             raise ValueError(
@@ -102,7 +101,6 @@ class DenseCellArray(CellArray):
         if self.ndim == 2 and data.shape[1] != self.shape[1]:
             raise ValueError(f"Data columns {data.shape[1]} don't match array columns {self.shape[1]}.")
 
-        # Construct write region
         if self.ndim == 1:
             write_region = slice(start_row, end_row)
         else:  # 2D
@@ -110,4 +108,5 @@ class DenseCellArray(CellArray):
 
         # write_data = {self._attr: data} if len(self.attr_names) > 1 else data
         with self.open_array(mode="w") as array:
+            print("write_region", write_region)
             array[write_region] = data

{cellarr_array-0.1.0/src/cellarr_array → cellarr_array-0.3.0/src/cellarr_array/core}/helpers.py

@@ -8,7 +8,7 @@ from typing import List, Optional, Tuple, Union
 import numpy as np
 import tiledb
 
-from .config import CellArrConfig
+from ..utils.config import CellArrConfig
 
 __author__ = "Jayaram Kancherla"
 __copyright__ = "Jayaram Kancherla"
@@ -52,7 +52,7 @@ def create_cellarray(
             Optional list of dimension names.
 
         dim_dtypes:
-            Optional list of dimension dtypes.
+            Optional list of dimension dtypes. Defaults to numpy's uint32.
 
         attr_name:
             Name of the data attribute.
@@ -67,29 +67,28 @@ def create_cellarray(
         ValueError: If dimensions are invalid or inputs are inconsistent.
     """
     config = config or CellArrConfig()
+    tiledb_ctx = tiledb.Config(config.ctx_config) if config.ctx_config else None
 
     if attr_dtype is None:
         attr_dtype = np.float32
     if isinstance(attr_dtype, str):
         attr_dtype = np.dtype(attr_dtype)
 
-    # Require either shape or dim_dtypes
     if shape is None and dim_dtypes is None:
         raise ValueError("Either 'shape' or 'dim_dtypes' must be provided.")
 
     if shape is not None:
         if len(shape) not in (1, 2):
-            raise ValueError("Only 1D and 2D arrays are supported.")
+            raise ValueError("Shape must have 1 or 2 dimensions.")
 
     # Set dimension dtypes, defaults to numpy uint32
     if dim_dtypes is None:
         dim_dtypes = [np.uint32] * len(shape)
     else:
         if len(dim_dtypes) not in (1, 2):
-            raise ValueError("Only 1D and 2D arrays are supported.")
+            raise ValueError("Array must have 1 or 2 dimensions.")
         dim_dtypes = [np.dtype(dt) if isinstance(dt, str) else dt for dt in dim_dtypes]
 
-    # Calculate shape from dtypes if needed
     if shape is None:
         shape = tuple(np.iinfo(dt).max if np.issubdtype(dt, np.integer) else None for dt in dim_dtypes)
     if None in shape:
@@ -97,7 +96,6 @@ def create_cellarray(
             np.iinfo(dt).max if s is None and np.issubdtype(dt, np.integer) else s for s, dt in zip(shape, dim_dtypes)
         )
 
-    # Set dimension names
     if dim_names is None:
         dim_names = [f"dim_{i}" for i in range(len(shape))]
 
@@ -105,42 +103,53 @@ def create_cellarray(
     if not (len(shape) == len(dim_dtypes) == len(dim_names)):
         raise ValueError("Lengths of 'shape', 'dim_dtypes', and 'dim_names' must match.")
 
-    dom = tiledb.Domain(
-        *[
-            tiledb.Dim(name=name, domain=(0, s - 1), tile=min(s, config.tile_capacity), dtype=dt)
-            for name, s, dt in zip(dim_names, shape, dim_dtypes)
-        ],
-        ctx=tiledb.Ctx(config.ctx_config),
-    )
+    dims = []
+    for name, s, dt in zip(dim_names, shape, dim_dtypes):
+        if np.issubdtype(dt, np.integer):
+            domain = (0, 0 if s == 0 else s - 1)
+            tile = min(1 if s == 0 else s // 2, config.tile_capacity // 2)
+            dim_dtype = dt
+        else:  # Assumes string or object dtype
+            domain = (None, None)
+            tile = None
+            dim_dtype = "ascii"
+
+        dims.append(
+            tiledb.Dim(
+                name=name,
+                domain=domain,
+                tile=tile,
+                dtype=dim_dtype,
+            )
+        )
 
-    attr = tiledb.Attr(
+    dom = tiledb.Domain(*dims, ctx=tiledb_ctx)
+    attr_obj = tiledb.Attr(
         name=attr_name,
         dtype=attr_dtype,
         filters=config.attrs_filters.get(attr_name, config.attrs_filters.get("", None)),
+        ctx=tiledb_ctx,
     )
-
     schema = tiledb.ArraySchema(
         domain=dom,
-        attrs=[attr],
+        attrs=[attr_obj],
         cell_order=config.cell_order,
         tile_order=config.tile_order,
         sparse=sparse,
         coords_filters=config.coords_filters,
         offsets_filters=config.offsets_filters,
-        ctx=tiledb.Ctx(config.ctx_config),
+        ctx=tiledb_ctx,
     )
-
-    tiledb.Array.create(uri, schema)
+    tiledb.Array.create(uri, schema, ctx=tiledb_ctx)
 
     # Import here to avoid circular imports
-    from .cellarray_dense import DenseCellArray
-    from .cellarray_sparse import SparseCellArray
+    from .dense import DenseCellArray
+    from .sparse import SparseCellArray
 
-    # Return appropriate array type
     return (
-        SparseCellArray(uri=uri, attr=attr_name, mode=mode)
+        SparseCellArray(uri=uri, attr=attr_name, mode=mode, config_or_context=tiledb_ctx)
         if sparse
-        else DenseCellArray(uri=uri, attr=attr_name, mode=mode)
+        else DenseCellArray(uri=uri, attr=attr_name, mode=mode, config_or_context=tiledb_ctx)
     )
 
 
@@ -148,59 +157,100 @@ class SliceHelper:
     """Helper class for handling array slicing operations."""
 
     @staticmethod
-    def is_contiguous_indices(indices: List[int]) -> Optional[slice]:
-        """Check if indices can be represented as a contiguous slice."""
+    def is_contiguous_indices(indices: List) -> Optional[slice]:
+        """Checks if a list of indices is contiguous and can be converted to a slice.
+
+        Returns None if the list is not contiguous or contains non-integers.
+        """
         if not indices:
             return None
 
-        diffs = np.diff(indices)
+        if not all(isinstance(i, (int, np.integer)) for i in indices):
+            return None
+
+        sorted_indices = sorted(list(set(indices)))
+        if not sorted_indices:
+            return None
+
+        if len(sorted_indices) == 1:
+            return slice(sorted_indices[0], sorted_indices[0] + 1, None)
+
+        diffs = np.diff(sorted_indices)
         if np.all(diffs == 1):
-            return slice(indices[0], indices[-1] + 1, None)
+            return slice(sorted_indices[0], sorted_indices[-1] + 1, None)
+
         return None
 
     @staticmethod
-    def normalize_index(idx: Union[int, slice, List[int]], dim_size: int) -> Union[slice, List[int], EllipsisType]:
+    def normalize_index(
+        idx: Union[int, range, slice, List, str, EllipsisType],
+        dim_size: int,
+        dim_dtype: np.dtype,
+    ):
         """Normalize index to handle negative indices and ensure consistency."""
+        is_string_dim = np.issubdtype(dim_dtype, np.str_) or np.issubdtype(dim_dtype, np.bytes_)
+
+        if is_string_dim:
+            if isinstance(idx, (str, bytes)):
+                return [idx]
+            if isinstance(idx, list) and all(isinstance(i, (str, bytes)) for i in idx):
+                return idx
+            if isinstance(idx, slice):
+                # For string dimensions, we do not normalize the slice with integer sizes
+                return idx
+            if isinstance(idx, EllipsisType):
+                return idx
+            raise TypeError(f"Unsupported index type '{type(idx).__name__}' for string dimension.")
 
         if isinstance(idx, EllipsisType):
             return idx
 
-        # Convert ranges to slices
         if isinstance(idx, range):
             idx = slice(idx.start, idx.stop, idx.step)
 
         if isinstance(idx, slice):
-            start = idx.start if idx.start is not None else 0
-            stop = idx.stop if idx.stop is not None else dim_size
-            step = idx.step
+            start, stop, step = idx.start, idx.stop, idx.step
+
+            # Resolve None to full dimension slice parts
+            if start is None:
+                start = 0
+
+            if stop is None:
+                stop = dim_size
 
             # Handle negative indices
             if start < 0:
-                start = dim_size + start
-
+                start += dim_size
             if stop < 0:
-                stop = dim_size + stop
-
-            if start < 0 or start > dim_size:
-                raise IndexError(f"Start index {start} out of bounds for dimension size {dim_size}")
-            if stop < 0 or stop > dim_size:
-                raise IndexError(f"Stop index {stop} out of bounds for dimension size {dim_size}")
+                stop += dim_size
 
+            # Clamping slice arguments to dimensions
+            stop = min(stop, dim_size)
+            start = max(0, start)
             return slice(start, stop, step)
 
-        elif isinstance(idx, list):
+        if isinstance(idx, list):
+            if not idx:
+                return []
+            # This check only applies to integer lists
+            if not all(isinstance(i, (int, np.integer)) for i in idx):
+                raise TypeError("List indices must be integers for numeric dimensions.")
+
             norm_idx = [i if i >= 0 else dim_size + i for i in idx]
             if any(i < 0 or i >= dim_size for i in norm_idx):
-                raise IndexError(f"List indices {idx} out of bounds for dimension size {dim_size}")
-            return norm_idx
-
-        else:  # Single integer index
-            norm_idx = idx if idx >= 0 else dim_size + idx
-
-            if norm_idx < 0 or norm_idx >= dim_size:
-                raise IndexError(f"Index {idx} out of bounds for dimension size {dim_size}")
+                raise IndexError("List indices out of bounds for dimension size.")
+            return sorted(list(set(norm_idx)))
+
+        if isinstance(idx, (int, np.integer)):
+            norm_idx = int(idx)
+            if norm_idx < 0:
+                norm_idx += dim_size
+            if not (0 <= norm_idx < dim_size):
+                raise IndexError(f"Index {idx} out of bounds for dimension size.")
             return slice(norm_idx, norm_idx + 1, None)
 
+        raise TypeError(f"Index type {type(idx)} not supported for normalization.")
+
 
 def create_group(output_path, group_name):
     tiledb.group_create(f"{output_path}/{group_name}")

cellarr_array-0.1.0/src/cellarr_array/cellarray_sparse.py → cellarr_array-0.3.0/src/cellarr_array/core/sparse.py

@@ -9,8 +9,8 @@ import numpy as np
 import tiledb
 from scipy import sparse
 
-from .cellarray_base import CellArray
 from .helpers import SliceHelper
+from .base import CellArray
 
 __author__ = "Jayaram Kancherla"
 __copyright__ = "Jayaram Kancherla"
@@ -28,7 +28,7 @@ class SparseCellArray(CellArray):
         mode: Optional[Literal["r", "w", "d", "m"]] = None,
         config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None,
         return_sparse: bool = True,
-        sparse_coerce: Union[sparse.csr_matrix, sparse.csc_matrix] = sparse.csr_matrix,
+        sparse_format: Union[sparse.csr_matrix, sparse.csc_matrix] = sparse.csr_matrix,
         validate: bool = True,
         **kwargs,
     ):
@@ -66,7 +66,7 @@ class SparseCellArray(CellArray):
                 Whether to return a sparse representation of the data when object is sliced.
                 Default is to return a dictionary that contains coordinates and values.
 
-            sparse_coerce:
+            sparse_format:
                 Format to return, defaults to csr_matrix.
 
             validate:
@@ -86,7 +86,7 @@ class SparseCellArray(CellArray):
         )
 
         self.return_sparse = return_sparse
-        self.sparse_coerce = sparse.csr_matrix if sparse_coerce is None else sparse_coerce
+        self.sparse_format = sparse.csr_matrix if sparse_format is None else sparse_format
 
     def _validate_matrix_dims(self, data: sparse.spmatrix) -> Tuple[sparse.coo_matrix, bool]:
         """Validate and adjust matrix dimensions if needed.
@@ -126,7 +126,7 @@ class SparseCellArray(CellArray):
                 shape.append(idx.stop - (idx.start or 0))
             elif isinstance(idx, list):
                 shape.append(len(set(idx)))
-            else:  # single integer
+            else:
                 shape.append(1)
 
         # Always return (n,1) shape for CSR matrix
@@ -140,20 +140,16 @@ class SparseCellArray(CellArray):
         """Convert TileDB result to CSR format or dense array."""
         data = result[self._attr]
 
-        # empty result
         if len(data) == 0:
-            print("is emoty")
             if not self.return_sparse:
                 return result
             else:
-                # For COO output, return empty sparse matrix
                 if self.ndim == 1:
-                    matrix = self.sparse_coerce((1, shape[0]))
+                    matrix = self.sparse_format((1, shape[0]))
                     return matrix[:, key[0]]
 
-                return self.sparse_coerce(shape)[key]
+                return self.sparse_format(shape)[key]
 
-        # Get coordinates
         coords = []
         for dim_name in self.dim_names:
             dim_coords = result[dim_name]
@@ -164,11 +160,12 @@ class SparseCellArray(CellArray):
             coords = [np.zeros_like(coords[0]), coords[0]]
             shape = (1, shape[0])
 
-        # Create sparse matrix
         matrix = sparse.coo_matrix((data, tuple(coords)), shape=shape)
-        if self.sparse_coerce in (sparse.csr_matrix, sparse.csr_array):
+
+        sliced = matrix
+        if self.sparse_format in (sparse.csr_matrix, sparse.csr_array):
             sliced = matrix.tocsr()
-        elif self.sparse_coerce in (sparse.csc_matrix, sparse.csc_array):
+        elif self.sparse_format in (sparse.csc_matrix, sparse.csc_array):
             sliced = matrix.tocsc()
 
         if self.ndim == 1:
@@ -200,7 +197,6 @@ class SparseCellArray(CellArray):
         if all(isinstance(idx, slice) for idx in optimized_key):
             return self._direct_slice(tuple(optimized_key))
 
-        # For mixed slice-list queries, adjust slice bounds
         tiledb_key = []
         for idx in key:
             if isinstance(idx, slice):
@@ -239,10 +235,8 @@ class SparseCellArray(CellArray):
         if not sparse.issparse(data):
             raise TypeError("Input must be a scipy sparse matrix.")
 
-        # Validate and adjust dimensions
         coo_data, is_1d = self._validate_matrix_dims(data)
 
-        # Check bounds
         end_row = start_row + coo_data.shape[0]
         if end_row > self.shape[0]:
             raise ValueError(

cellarr_array-0.3.0/src/cellarr_array/dataloaders/__init__.py

@@ -0,0 +1,3 @@
+from .denseloader import DenseArrayDataset, construct_dense_array_dataloader
+from .iterabledataloader import CellArrayIterableDataset, construct_iterable_dataloader
+from .sparseloader import SparseArrayDataset, construct_sparse_array_dataloader