cellarr-array 0.0.1__py3-none-any.whl

cellarr_array/CellArray.py

@@ -0,0 +1,236 @@
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from typing import List, Literal, Optional, Tuple, Union
+
+import numpy as np
+import tiledb
+from scipy import sparse
+
+from .config import ConsolidationConfig
+from .helpers import SliceHelper
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+class CellArray(ABC):
+    """Abstract base class for TileDB array operations."""
+
+    def __init__(
+        self,
+        uri: str,
+        attr: str = "data",
+        mode: Optional[Literal["r", "w", "n", "d"]] = None,
+        config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None,
+        validate: bool = True,
+    ):
+        """Initialize the object.
+
+        Args:
+            uri:
+                URI to the array.
+
+            attr:
+                Attribute to access.
+                Defaults to "data".
+
+            mode:
+                Open the array object in read 'r', write 'w', modify
+                exclusive 'm' mode, or delete 'd' mode.
+
+                Defaults to None for automatic mode switching.
+
+            config_or_context:
+                Config or context object.
+                Defaults to None.
+
+            validate:
+                Whether to validate the attributes.
+                Defaults to True.
+        """
+        self.uri = uri
+        self._mode = mode
+
+        if config_or_context is None:
+            config_or_context = tiledb.Config()
+
+        if isinstance(config_or_context, tiledb.Config):
+            ctx = tiledb.Ctx(config_or_context)
+        elif isinstance(config_or_context, tiledb.Ctx):
+            ctx = config_or_context
+        else:
+            raise TypeError("'config_or_context' must be either TileDB config or a context object.")
+
+        self._ctx = ctx
+        self._array = None
+        self._shape = None
+        self._ndim = None
+        self._dim_names = None
+        self._attr_names = None
+
+        if validate:
+            self._validate(attr=attr)
+
+        self._attr = attr
+
+    def _validate(self, attr):
+        with self.open_array(mode="r") as A:
+            if A.ndim > 2:
+                raise ValueError("Only 1D and 2D arrays are supported.")
+
+            if attr not in self.attr_names:
+                raise ValueError(
+                    f"Attribute '{attr}' does not exist in the array. Available attributes: {self.attr_names}."
+                )
+
+    @property
+    def mode(self) -> Optional[str]:
+        """Get current array mode."""
+        return self._mode
+
+    @mode.setter
+    def mode(self, value: Optional[str]):
+        """Set array mode.
+
+        Args:
+            value:
+                One of `None`, 'r', 'w', or 'm', 'd'.
+        """
+        if value is not None and value not in ["r", "w", "m", "d"]:
+            raise ValueError("Mode must be one of: None, 'r', 'w', 'm', 'd'")
+        self._mode = value
+
+    @property
+    def dim_names(self) -> List[str]:
+        """Get dimension names of the array."""
+        if self._dim_names is None:
+            with self.open_array(mode="r") as A:
+                self._dim_names = [dim.name for dim in A.schema.domain]
+        return self._dim_names
+
+    @property
+    def attr_names(self) -> List[str]:
+        """Get attribute names of the array."""
+        if self._attr_names is None:
+            with self.open_array(mode="r") as A:
+                self._attr_names = [A.schema.attr(i).name for i in range(A.schema.nattr)]
+        return self._attr_names
+
+    @property
+    def shape(self) -> Tuple[int, ...]:
+        """Get array shape from schema domain."""
+        if self._shape is None:
+            with self.open_array(mode="r") as A:
+                self._shape = tuple(int(dim.domain[1] - dim.domain[0] + 1) for dim in A.schema.domain)
+        return self._shape
+
+    @property
+    def nonempty_domain(self) -> Tuple[int, ...]:
+        """Get array non-empty domain."""
+        if self._nonempty_domain is None:
+            with self.open_array(mode="r") as A:
+                self._nonempty_domain = A.nonempty_domain()
+        return self._nonempty_domain
+
+    @property
+    def ndim(self) -> int:
+        """Get number of dimensions."""
+        if self._ndim is None:
+            self._ndim = len(self.shape)
+        return self._ndim
+
+    @contextmanager
+    def open_array(self, mode: Optional[str] = None):
+        """Context manager for array operations.
+
+        Args:
+            mode:
+                Override mode for this operation.
+        """
+        mode = mode if mode is not None else self.mode
+        mode = mode if mode is not None else "r"  # Default to read mode
+
+        array = tiledb.open(self.uri, mode=mode, ctx=self._ctx)
+        try:
+            yield array
+        finally:
+            array.close()
+
+    def __getitem__(self, key: Union[slice, Tuple[Union[slice, List[int]], ...]]):
+        """Get item implementation that routes to either direct slicing or multi_index
+        based on the type of indices provided.
+
+        Args:
+            key:
+                Slice or list of indices for each dimension in the array.
+        """
+        if not isinstance(key, tuple):
+            key = (key,)
+
+        if len(key) > self.ndim:
+            raise IndexError(f"Invalid number of dimensions: got {len(key)}, expected {self.ndim}")
+
+        # Normalize all indices
+        normalized_key = tuple(SliceHelper.normalize_index(idx, self.shape[i]) for i, idx in enumerate(key))
+
+        # Check if we can use direct slicing
+        use_direct = all(isinstance(idx, slice) for idx in normalized_key)
+
+        if use_direct:
+            return self._direct_slice(normalized_key)
+        else:
+            return self._multi_index(normalized_key)
+
+    @abstractmethod
+    def _direct_slice(self, key: Tuple[slice, ...]) -> np.ndarray:
+        """Implementation for direct slicing."""
+        pass
+
+    @abstractmethod
+    def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray:
+        """Implementation for multi-index access."""
+        pass
+
+    def vacuum(self) -> None:
+        """Remove deleted fragments from the array."""
+        tiledb.vacuum(self.uri)
+
+    def consolidate(self, config: Optional[ConsolidationConfig] = None) -> None:
+        """Consolidate array fragments.
+
+        Args:
+            config:
+                Optional consolidation configuration.
+        """
+        if config is None:
+            config = ConsolidationConfig()
+
+        consolidation_cfg = tiledb.Config()
+
+        consolidation_cfg["sm.consolidation.steps"] = config.steps
+        consolidation_cfg["sm.consolidation.step_min_frags"] = config.step_min_frags
+        consolidation_cfg["sm.consolidation.step_max_frags"] = config.step_max_frags
+        consolidation_cfg["sm.consolidation.buffer_size"] = config.buffer_size
+        consolidation_cfg["sm.mem.total_budget"] = config.total_budget
+
+        tiledb.consolidate(self.uri, config=consolidation_cfg)
+
+        if config.vacuum_after:
+            self.vacuum()
+
+    @abstractmethod
+    def write_batch(self, data: Union[np.ndarray, sparse.spmatrix], start_row: int, **kwargs) -> None:
+        """Write a batch of data to the array starting at the specified row.
+
+        Args:
+            data:
+                Data to write (numpy array for dense, scipy sparse matrix for sparse).
+
+            start_row:
+                Starting row index for writing.
+
+            **kwargs:
+                Additional arguments for write operation.
+        """
+        pass

cellarr_array/DenseCellArray.py

@@ -0,0 +1,108 @@
+from typing import List, Tuple, Union
+
+import numpy as np
+
+from .CellArray import CellArray
+from .helpers import SliceHelper
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+class DenseCellArray(CellArray):
+    """Implementation for dense TileDB arrays."""
+
+    def _direct_slice(self, key: Tuple[slice, ...]) -> np.ndarray:
+        """Implementation for direct slicing of dense arrays.
+
+        Args:
+            key:
+                Tuple of slice objects.
+
+        Returns:
+            Sliced data.
+        """
+        with self.open_array(mode="r") as array:
+            res = array[key]
+            return res[self._attr] if self._attr is not None else res
+
+    def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray:
+        """Implementation for multi-index access of dense arrays.
+
+        Args:
+            key:
+                Tuple of slice objects or index lists.
+
+        Returns:
+            Sliced data.
+        """
+        # Try to optimize contiguous indices to slices
+        optimized_key = []
+        for idx in key:
+            if isinstance(idx, list):
+                slice_idx = SliceHelper.is_contiguous_indices(idx)
+                optimized_key.append(slice_idx if slice_idx is not None else idx)
+            else:
+                optimized_key.append(idx)
+
+        # If all indices are now slices, use direct slicing
+        if all(isinstance(idx, slice) for idx in optimized_key):
+            return self._direct_slice(tuple(optimized_key))
+
+        # For mixed slice-list queries, adjust slice bounds to exclude upper bound
+        tiledb_key = []
+        for idx in key:
+            if isinstance(idx, slice):
+                # Adjust stop to be exclusive by subtracting 1 if stop is not None
+                stop = None if idx.stop is None else idx.stop - 1
+                tiledb_key.append(slice(idx.start, stop, idx.step))
+            else:
+                tiledb_key.append(idx)
+
+        with self.open_array(mode="r") as array:
+            res = array.multi_index[tuple(tiledb_key)]
+            return res[self._attr] if self._attr is not None else res
+
+    def write_batch(self, data: np.ndarray, start_row: int, **kwargs) -> None:
+        """Write a batch of data to the dense array.
+
+        Args:
+            data:
+                Numpy array to write.
+
+            start_row:
+                Starting row index for writing.
+
+            **kwargs:
+                Additional arguments passed to TileDB write operation.
+
+        Raises:
+            TypeError: If input is not a numpy array.
+            ValueError: If dimensions don't match or bounds are exceeded.
+        """
+        if not isinstance(data, np.ndarray):
+            raise TypeError("Input must be a numpy array.")
+
+        if len(data.shape) != self.ndim:
+            raise ValueError(f"Data dimensions {data.shape} don't match array dimensions {self.shape}.")
+
+        # Check bounds
+        end_row = start_row + data.shape[0]
+        if end_row > self.shape[0]:
+            raise ValueError(
+                f"Write operation would exceed array bounds. End row {end_row} > array rows {self.shape[0]}."
+            )
+
+        if self.ndim == 2 and data.shape[1] != self.shape[1]:
+            raise ValueError(f"Data columns {data.shape[1]} don't match array columns {self.shape[1]}.")
+
+        # Construct write region
+        if self.ndim == 1:
+            write_region = slice(start_row, end_row)
+        else:  # 2D
+            write_region = (slice(start_row, end_row), slice(0, self.shape[1]))
+
+        # write_data = {self._attr: data} if len(self.attr_names) > 1 else data
+        with self.open_array(mode="w") as array:
+            array[write_region] = data

cellarr_array/SparseCellArray.py

@@ -0,0 +1,202 @@
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import tiledb
+from scipy import sparse
+
+from .CellArray import CellArray
+from .helpers import SliceHelper
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+class SparseCellArray(CellArray):
+    """Implementation for sparse TileDB arrays."""
+
+    def __init__(
+        self,
+        uri: str,
+        attr: str = "data",
+        mode: str = None,
+        config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None,
+        return_sparse: bool = True,
+        sparse_coerce: Union[sparse.csr_matrix, sparse.csc_matrix] = sparse.csr_matrix,
+    ):
+        """Initialize SparseCellArray."""
+        super().__init__(uri, attr, mode, config_or_context)
+
+        self.return_sparse = return_sparse
+        self.sparse_coerce = sparse.csr_matrix if sparse_coerce is None else sparse_coerce
+
+    def _validate_matrix_dims(self, data: sparse.spmatrix) -> Tuple[sparse.coo_matrix, bool]:
+        """Validate and adjust matrix dimensions if needed.
+
+        Args:
+            data:
+                Input sparse matrix.
+
+        Returns:
+            Tuple of (adjusted matrix, is_1d flag).
+
+        Raises:
+            ValueError: If dimensions are incompatible.
+        """
+        coo_data = data.tocoo() if not isinstance(data, sparse.coo_matrix) else data
+
+        is_1d = self.ndim == 1
+        if is_1d:
+            if coo_data.shape[0] == 1:
+                # Convert (1,N) to (N,1)
+                coo_data = sparse.coo_matrix(
+                    (coo_data.data, (coo_data.col, np.zeros_like(coo_data.col))), shape=(coo_data.shape[1], 1)
+                )
+            elif coo_data.shape[1] != 1:
+                raise ValueError(f"1D array expects (N, 1) matrix, got {coo_data.shape}")
+
+        return coo_data, is_1d
+
+    def _get_slice_shape(self, key: Tuple[Union[slice, List[int]], ...]) -> Tuple[int, ...]:
+        """Calculate shape of sliced result.
+
+        Always returns 2D shape (n,1) for csr matrix compatibility.
+        """
+        shape = []
+        for i, idx in enumerate(key):
+            if isinstance(idx, slice):
+                shape.append(idx.stop - (idx.start or 0))
+            elif isinstance(idx, list):
+                shape.append(len(set(idx)))
+            else:  # single integer
+                shape.append(1)
+
+        # Always return (n,1) shape for CSR matrix
+        if self.ndim == 1:
+            return (shape[0], 1)
+        return tuple(shape)
+
+    def _to_sparse_format(
+        self, result: Dict[str, np.ndarray], key: Tuple[Union[slice, List[int]], ...], shape: Tuple[int, ...]
+    ) -> Union[np.ndarray, sparse.spmatrix]:
+        """Convert TileDB result to CSR format or dense array."""
+        data = result[self._attr]
+
+        # empty result
+        if len(data) == 0:
+            print("is emoty")
+            if not self.return_sparse:
+                return result
+            else:
+                # For COO output, return empty sparse matrix
+                if self.ndim == 1:
+                    matrix = self.sparse_coerce((1, shape[0]))
+                    return matrix[:, key[0]]
+
+                return self.sparse_coerce(shape)[key]
+
+        # Get coordinates
+        coords = []
+        for dim_name in self.dim_names:
+            dim_coords = result[dim_name]
+            coords.append(dim_coords)
+
+        # For 1D arrays, add zero column coordinates, also (N, 1)
+        if self.ndim == 1:
+            coords = [np.zeros_like(coords[0]), coords[0]]
+            shape = (1, shape[0])
+
+        # Create sparse matrix
+        matrix = sparse.coo_matrix((data, tuple(coords)), shape=shape)
+        if self.sparse_coerce in (sparse.csr_matrix, sparse.csr_array):
+            sliced = matrix.tocsr()
+        elif self.sparse_coerce in (sparse.csc_matrix, sparse.csc_array):
+            sliced = matrix.tocsc()
+
+        if self.ndim == 1:
+            return sliced[:, key[0]]
+
+        return sliced[key]
+
+    def _direct_slice(self, key: Tuple[slice, ...]) -> Union[np.ndarray, sparse.coo_matrix]:
+        """Implementation for direct slicing of sparse arrays."""
+        with self.open_array(mode="r") as array:
+            result = array[key]
+
+            if not self.return_sparse:
+                return result
+
+            return self._to_sparse_format(result, key, self.shape)
+
+    def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> Union[np.ndarray, sparse.coo_matrix]:
+        """Implementation for multi-index access of sparse arrays."""
+        # Try to optimize contiguous indices to slices
+        optimized_key = []
+        for idx in key:
+            if isinstance(idx, list):
+                slice_idx = SliceHelper.is_contiguous_indices(idx)
+                optimized_key.append(slice_idx if slice_idx is not None else idx)
+            else:
+                optimized_key.append(idx)
+
+        if all(isinstance(idx, slice) for idx in optimized_key):
+            return self._direct_slice(tuple(optimized_key))
+
+        # For mixed slice-list queries, adjust slice bounds
+        tiledb_key = []
+        for idx in key:
+            if isinstance(idx, slice):
+                stop = None if idx.stop is None else idx.stop - 1
+                tiledb_key.append(slice(idx.start, stop, idx.step))
+            else:
+                tiledb_key.append(idx)
+
+        with self.open_array(mode="r") as array:
+            result = array.multi_index[tuple(tiledb_key)]
+
+            if not self.return_sparse:
+                return result
+
+            return self._to_sparse_format(result, key, self.shape)
+
+    def write_batch(
+        self, data: Union[sparse.spmatrix, sparse.csc_matrix, sparse.coo_matrix], start_row: int, **kwargs
+    ) -> None:
+        """Write a batch of sparse data to the array.
+
+        Args:
+            data:
+                Scipy sparse matrix (CSR, CSC, or COO format).
+
+            start_row:
+                Starting row index for writing.
+
+            **kwargs:
+                Additional arguments passed to TileDB write operation.
+
+        Raises:
+            TypeError: If input is not a sparse matrix.
+            ValueError: If dimensions don't match or bounds are exceeded.
+        """
+        if not sparse.issparse(data):
+            raise TypeError("Input must be a scipy sparse matrix.")
+
+        # Validate and adjust dimensions
+        data, is_1d = self._validate_matrix_dims(data)
+
+        # Check bounds
+        end_row = start_row + data.shape[0]
+        if end_row > self.shape[0]:
+            raise ValueError(
+                f"Write operation would exceed array bounds. End row {end_row} > array rows {self.shape[0]}."
+            )
+
+        if not is_1d and data.shape[1] != self.shape[1]:
+            raise ValueError(f"Data columns {data.shape[1]} don't match array columns {self.shape[1]}.")
+
+        adjusted_rows = data.row + start_row
+        with self.open_array(mode="w") as array:
+            if is_1d:
+                array[adjusted_rows] = data.data
+            else:
+                array[adjusted_rows, data.col] = data.data

cellarr_array/__init__.py

@@ -0,0 +1,21 @@
+import sys
+
+if sys.version_info[:2] >= (3, 8):
+    # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8`
+    from importlib.metadata import PackageNotFoundError, version  # pragma: no cover
+else:
+    from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = "cellarr-array"
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError
+
+from .config import CellArrConfig, ConsolidationConfig
+from .DenseCellArray import DenseCellArray
+from .SparseCellArray import SparseCellArray
+from .helpers import create_cellarray, SliceHelper

cellarr_array/config.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Union
+
+import tiledb
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+@dataclass
+class CellArrConfig:
+    """Configuration class for TileDB array creation and access."""
+
+    tile_capacity: int = 100000
+    cell_order: str = "row-major"
+    tile_order: str = "row-major"
+    coords_filters: List[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()])
+    offsets_filters: List[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()])
+    attrs_filters: Dict[str, List[tiledb.Filter]] = field(default_factory=lambda: {"": [tiledb.LZ4Filter()]})
+    ctx_config: Dict[str, Any] = field(default_factory=dict)
+
+    @staticmethod
+    def create_filter(filter_config: Union[Dict[str, Any], tiledb.Filter]) -> tiledb.Filter:
+        """Create a TileDB Filter object from configuration."""
+        if isinstance(filter_config, tiledb.Filter):
+            return filter_config
+
+        if isinstance(filter_config, dict):
+            filter_name = filter_config.get("name", "").lower()
+            filter_level = filter_config.get("level", None)
+
+            if filter_name == "zstd":
+                return tiledb.ZstdFilter(level=filter_level)
+            elif filter_name == "gzip":
+                return tiledb.GzipFilter(level=filter_level)
+            elif filter_name == "bzip2":
+                return tiledb.Bzip2Filter(level=filter_level)
+            elif filter_name == "double-delta":
+                return tiledb.DoubleDeltaFilter()
+            elif filter_name == "bit-width-reduction":
+                return tiledb.BitWidthReductionFilter()
+            else:
+                raise ValueError(f"Unsupported filter type: {filter_name}")
+
+        raise TypeError("Filter must be either a TileDB Filter object or a configuration dictionary")
+
+    def __post_init__(self):
+        """Convert filter configurations to TileDB Filter objects."""
+        if not isinstance(self.coords_filters, list):
+            self.coords_filters = [self.coords_filters]
+        self.coords_filters = [self.create_filter(f) for f in self.coords_filters]
+
+        if not isinstance(self.offsets_filters, list):
+            self.offsets_filters = [self.offsets_filters]
+        self.offsets_filters = [self.create_filter(f) for f in self.offsets_filters]
+
+        for attr, filters in self.attrs_filters.items():
+            if not isinstance(filters, list):
+                filters = [filters]
+            self.attrs_filters[attr] = [self.create_filter(f) for f in filters]
+
+
+@dataclass
+class ConsolidationConfig:
+    """Configuration for array consolidation."""
+
+    steps: int = 100000
+    step_min_frags: int = 2
+    step_max_frags: int = 10
+    buffer_size: int = 15000000000  # 15GB
+    total_budget: int = 40000000000  # 40GB
+    num_threads: int = 4
+    vacuum_after: bool = True

cellarr_array/helpers.py

@@ -0,0 +1,194 @@
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import tiledb
+
+from .config import CellArrConfig
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def create_cellarray(
+    uri: str,
+    shape: Optional[Tuple[Optional[int], ...]] = None,
+    attr_dtype: Optional[Union[str, np.dtype]] = None,
+    sparse: bool = False,
+    mode: str = None,
+    config: Optional[CellArrConfig] = None,
+    dim_names: Optional[List[str]] = None,
+    dim_dtypes: Optional[List[Union[str, np.dtype]]] = None,
+    attr_name: str = "data",
+    **kwargs,
+):
+    """Factory function to create a new TileDB cell array.
+
+    Args:
+        uri:
+            Array URI.
+
+        shape:
+            Optional array shape. If None or contains None, uses dtype max.
+
+        attr_dtype:
+            Data type for the attribute. Defaults to float32.
+
+        sparse:
+            Whether to create a sparse array.
+
+        mode:
+            Array open mode. Defaults to None for automatic switching.
+
+        config:
+            Optional configuration.
+
+        dim_names:
+            Optional list of dimension names.
+
+        dim_dtypes:
+            Optional list of dimension dtypes.
+
+        attr_name:
+            Name of the data attribute.
+
+        **kwargs:
+            Additional arguments for array creation.
+
+    Returns:
+        CellArray instance.
+
+    Raises:
+        ValueError: If dimensions are invalid or inputs are inconsistent.
+    """
+    config = config or CellArrConfig()
+
+    if attr_dtype is None:
+        attr_dtype = np.float32
+    if isinstance(attr_dtype, str):
+        attr_dtype = np.dtype(attr_dtype)
+
+    # Require either shape or dim_dtypes
+    if shape is None and dim_dtypes is None:
+        raise ValueError("Either 'shape' or 'dim_dtypes' must be provided.")
+
+    if shape is not None:
+        if len(shape) not in (1, 2):
+            raise ValueError("Only 1D and 2D arrays are supported.")
+
+    # Set dimension dtypes, defaults to numpy uint32
+    if dim_dtypes is None:
+        dim_dtypes = [np.uint32] * len(shape)
+    else:
+        if len(dim_dtypes) not in (1, 2):
+            raise ValueError("Only 1D and 2D arrays are supported.")
+        dim_dtypes = [np.dtype(dt) if isinstance(dt, str) else dt for dt in dim_dtypes]
+
+    # Calculate shape from dtypes if needed
+    if shape is None:
+        shape = tuple(np.iinfo(dt).max if np.issubdtype(dt, np.integer) else None for dt in dim_dtypes)
+    if None in shape:
+        shape = tuple(
+            np.iinfo(dt).max if s is None and np.issubdtype(dt, np.integer) else s for s, dt in zip(shape, dim_dtypes)
+        )
+
+    # Set dimension names
+    if dim_names is None:
+        dim_names = [f"dim_{i}" for i in range(len(shape))]
+
+    # Validate all input lengths
+    if not (len(shape) == len(dim_dtypes) == len(dim_names)):
+        raise ValueError("Lengths of 'shape', 'dim_dtypes', and 'dim_names' must match.")
+
+    dom = tiledb.Domain(
+        *[
+            tiledb.Dim(name=name, domain=(0, s - 1), tile=min(s, config.tile_capacity), dtype=dt)
+            for name, s, dt in zip(dim_names, shape, dim_dtypes)
+        ],
+        ctx=tiledb.Ctx(config.ctx_config),
+    )
+
+    attr = tiledb.Attr(
+        name=attr_name,
+        dtype=attr_dtype,
+        filters=config.attrs_filters.get(attr_name, config.attrs_filters.get("", None)),
+    )
+
+    schema = tiledb.ArraySchema(
+        domain=dom,
+        attrs=[attr],
+        cell_order=config.cell_order,
+        tile_order=config.tile_order,
+        sparse=sparse,
+        coords_filters=config.coords_filters,
+        offsets_filters=config.offsets_filters,
+        ctx=tiledb.Ctx(config.ctx_config),
+    )
+
+    tiledb.Array.create(uri, schema)
+
+    # Import here to avoid circular imports
+    from .DenseCellArray import DenseCellArray
+    from .SparseCellArray import SparseCellArray
+
+    # Return appropriate array type
+    return SparseCellArray(uri, attr=attr_name, mode=mode) if sparse else DenseCellArray(uri, attr=attr_name, mode=mode)
+
+
+class SliceHelper:
+    """Helper class for handling array slicing operations."""
+
+    @staticmethod
+    def is_contiguous_indices(indices: List[int]) -> Optional[slice]:
+        """Check if indices can be represented as a contiguous slice."""
+        if not indices:
+            return None
+
+        diffs = np.diff(indices)
+        if np.all(diffs == 1):
+            return slice(indices[0], indices[-1] + 1, None)
+        return None
+
+    @staticmethod
+    def normalize_index(idx: Union[int, slice, List[int]], dim_size: int) -> Union[slice, List[int]]:
+        """Normalize index to handle negative indices and ensure consistency."""
+
+        # Convert ranges to slices
+        if isinstance(idx, range):
+            idx = slice(idx.start, idx.stop, idx.step)
+
+        if isinstance(idx, slice):
+            start = idx.start if idx.start is not None else 0
+            stop = idx.stop if idx.stop is not None else dim_size
+            step = idx.step
+
+            # Handle negative indices
+            if start < 0:
+                start = dim_size + start
+
+            if stop < 0:
+                stop = dim_size + stop
+
+            if start < 0 or start > dim_size:
+                raise IndexError(f"Start index {start} out of bounds for dimension size {dim_size}")
+            if stop < 0 or stop > dim_size:
+                raise IndexError(f"Stop index {stop} out of bounds for dimension size {dim_size}")
+
+            return slice(start, stop, step)
+
+        elif isinstance(idx, list):
+            norm_idx = [i if i >= 0 else dim_size + i for i in idx]
+            if any(i < 0 or i >= dim_size for i in norm_idx):
+                raise IndexError(f"List indices {idx} out of bounds for dimension size {dim_size}")
+            return norm_idx
+
+        else:  # Single integer index
+            norm_idx = idx if idx >= 0 else dim_size + idx
+
+            if norm_idx < 0 or norm_idx >= dim_size:
+                raise IndexError(f"Index {idx} out of bounds for dimension size {dim_size}")
+            return slice(norm_idx, norm_idx + 1, None)
+
+
+def create_group(output_path, group_name):
+    tiledb.group_create(f"{output_path}/{group_name}")

cellarr_array-0.0.1.dist-info/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Jayaram Kancherla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cellarr_array-0.0.1.dist-info/METADATA

@@ -0,0 +1,161 @@
+Metadata-Version: 2.2
+Name: cellarr-array
+Version: 0.0.1
+Summary: Base class for handling TileDB backed arrays.
+Home-page: https://github.com/cellarr/cellarr-array
+Author: Jayaram Kancherla
+Author-email: jayaram.kancherla@gmail.com
+License: MIT
+Project-URL: Documentation, https://github.com/cellarr/cellarr-array
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+License-File: LICENSE.txt
+Requires-Dist: importlib-metadata; python_version < "3.8"
+Requires-Dist: tiledb
+Requires-Dist: numpy
+Requires-Dist: scipy
+Provides-Extra: testing
+Requires-Dist: setuptools; extra == "testing"
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
+
+[![PyPI-Server](https://img.shields.io/pypi/v/cellarr-array.svg)](https://pypi.org/project/cellarr-array/)
+![Unit tests](https://github.com/cellarr/cellarr-array/actions/workflows/run-tests.yml/badge.svg)
+
+# cellarr-array
+
+This package provided high-level wrappers for TileDB arrays optimized for handling genomic data matrices.
+
+## Install
+
+To get started, install the package from [PyPI](https://pypi.org/project/cellarr-array/)
+
+```bash
+pip install cellarr-array
+```
+
+## Quick Start
+
+### Creating Arrays
+
+```python
+import numpy as np
+from scipy import sparse
+from cellarr_array import create_cellarray, CellArrConfig
+
+# Create a dense 2D array
+dense_array = create_cellarray(
+    uri="dense_matrix.tdb",
+    shape=(10000, 5000),
+    attr_dtype=np.float32,
+    sparse=False,
+    dim_names=["cells", "genes"]
+)
+
+# Create a sparse 2D array with custom compression
+config = CellArrConfig(
+    tile_capacity=1000,
+    attrs_filters={"data": [{"name": "zstd", "level": 7}]}
+)
+sparse_array = create_cellarray(
+    uri="sparse_matrix.tdb",
+    shape=(10000, 5000),
+    attr_dtype=np.float32,
+    sparse=True,
+    config=config,
+    dim_names=["cells", "genes"]
+)
+
+# Create a 1D array
+array_1d = create_cellarray(
+    uri="vector.tdb",
+    shape=(1000,),
+    attr_dtype=np.float32,
+    sparse=False
+)
+```
+
+### Writing Data
+
+```python
+# Writing to dense arrays
+data = np.random.random((1000, 5000)).astype(np.float32)
+dense_array.write_batch(data, start_row=0)
+
+# Writing to sparse arrays
+sparse_data = sparse.random(1000, 5000, density=0.1, format="csr", dtype=np.float32)
+sparse_array.write_batch(sparse_data, start_row=0)
+
+# Writing to 1D arrays
+data_1d = np.random.random(100).astype(np.float32)
+array_1d.write_batch(data_1d, start_row=0)
+```
+
+### Reading Data
+
+```python
+# Slicing operations (similar to NumPy)
+
+# Full slice
+full_data = dense_array[:]
+
+# Partial slice
+subset = dense_array[100:200, 1000:2000]
+
+# Using lists of indices
+cells = [10, 20, 30]
+genes = [5, 15, 25]
+subset = dense_array[cells, genes]
+
+# Mixed slicing
+subset = dense_array[100:200, genes]
+```
+
+### Working with Sparse Arrays
+
+```python
+# Create a sparse array with COO output format
+coo_array = SparseCellArray(
+    uri="sparse_matrix.tdb",
+    return_coo=True
+)
+
+# Get result as COO matrix
+result = coo_array[100:200, 500:1000]
+
+# Result is scipy.sparse.coo_matrix
+assert sparse.isspmatrix_coo(result)
+
+# Perform sparse operations
+nnz = result.nnz
+density = result.nnz / (result.shape[0] * result.shape[1])
+
+# Convert to other sparse formats if needed
+result_csr = result.tocsr()
+```
+
+### Array Maintenance
+
+```python
+# Consolidate fragments
+array.consolidate()
+
+# Custom consolidation
+config = ConsolidationConfig(
+    steps=["fragment"],
+    vacuum_after=True
+)
+array.consolidate(config)
+
+# Vacuum
+array.vacuum()
+```
+
+<!-- biocsetup-notes -->
+
+## Note
+
+This project has been set up using [BiocSetup](https://github.com/biocpy/biocsetup)
+and [PyScaffold](https://pyscaffold.org/).

cellarr_array-0.0.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+cellarr_array/CellArray.py,sha256=vOaq-0FbVKeuS31992oc_N5IOBXclcVkczPNIbua5Ws,7498
+cellarr_array/DenseCellArray.py,sha256=iPrjFtGolnHB0BTi4A8ncEpoFI9FWe6oZHhA1Men3Wo,3745
+cellarr_array/SparseCellArray.py,sha256=8bajVOvUMaQhWU-_pZY0Cg9sD6kWRAJCu2G45uY-W4Q,7096
+cellarr_array/__init__.py,sha256=8m0_shRPKNNaNab5tGBL2l0K5XgkKCFuLAh7QGogfYo,778
+cellarr_array/config.py,sha256=67zBxpYY9N_v6TMdyljUIZmckbwOBcuLC99aJooGmfA,2917
+cellarr_array/helpers.py,sha256=O0RgDLIdYbWc01yp2Cw0EmjJ3g_uzlz2JnYE8W7PZEE,6182
+cellarr_array-0.0.1.dist-info/LICENSE.txt,sha256=qI2hRZobcUlj8gqFqXwqt522HeYyWvHLF00zCSZofHA,1084
+cellarr_array-0.0.1.dist-info/METADATA,sha256=UaSorFB0-5KuhVrM8pvdGuN98WQ6iSLgUUH6MtpJwXM,3747
+cellarr_array-0.0.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+cellarr_array-0.0.1.dist-info/top_level.txt,sha256=oErp0D8ABZV-QPtTiXT8_F2z36Ic7ykuDg_1Y84HLZM,14
+cellarr_array-0.0.1.dist-info/RECORD,,

cellarr_array-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cellarr_array-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+cellarr_array