cellarr-array 0.0.3__py3-none-any.whl → 0.2.0__py3-none-any.whl

cellarr_array/utils/mock.py

@@ -0,0 +1,167 @@
+import shutil
+from typing import Dict, Optional
+
+import numpy as np
+import scipy.sparse as sp
+import tiledb
+
+from ..core import DenseCellArray, SparseCellArray
+from ..core.helpers import CellArrConfig, create_cellarray
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def generate_tiledb_dense_array(
+    uri: str,
+    rows: int,
+    cols: int,
+    attr_name: str = "data",
+    attr_dtype: np.dtype = np.float32,
+    chunk_size: int = 1000,
+    tiledb_config: Optional[Dict] = None,
+):
+    """Generates a dense TileDB array and fills it with random float32 data.
+
+    Args:
+        uri:
+            URI for the new TileDB array.
+
+        rows:
+            Number of rows.
+
+        cols:
+            Number of columns (features).
+
+        attr_name:
+            Name of the attribute.
+
+        attr_dtype:
+            Data type of the attribute.
+
+        chunk_size:
+            Number of rows to write per batch.
+
+        tiledb_config:
+            TileDB context configuration.
+    """
+    if tiledb.array_exists(uri):
+        print(f"Array {uri} already exists. Removing.")
+        shutil.rmtree(uri)
+
+    print(f"Creating dense array at '{uri}' with shape ({rows}, {cols})")
+    cfg = CellArrConfig(ctx_config=tiledb_config if tiledb_config else {})
+
+    create_cellarray(
+        uri=uri,
+        shape=(rows, cols),
+        attr_dtype=attr_dtype,
+        sparse=False,
+        dim_names=["rows", "cols"],
+        attr_name=attr_name,
+        # config=cfg
+    )
+
+    ctx = tiledb.Ctx(cfg.ctx_config) if cfg.ctx_config else None
+    arr_writer = DenseCellArray(uri=uri, attr=attr_name, mode="w", config_or_context=ctx)
+
+    print("shape of writer", arr_writer.shape)
+
+    print(f"Writing data to dense array '{uri}'...")
+    for i in range(0, rows, chunk_size):
+        end_row = min(i + chunk_size, rows)
+        num_chunk_rows = end_row - i
+        data_chunk = np.random.rand(num_chunk_rows, cols).astype(attr_dtype)
+        print(i, end_row, num_chunk_rows, data_chunk.shape)
+        arr_writer.write_batch(data_chunk, start_row=i)
+        if (i // chunk_size) % 10 == 0:
+            print(f"  Dense write: {end_row}/{rows} rows written.")
+
+    print(f"Finished writing to dense array '{uri}'.")
+
+
+def generate_tiledb_sparse_array(
+    uri: str,
+    rows: int,
+    cols: int,
+    density: float = 0.01,
+    attr_name: str = "data",
+    attr_dtype: np.dtype = np.float32,
+    chunk_size: int = 1000,
+    tiledb_config: Optional[Dict] = None,
+    sparse_format_to_write="coo",
+):
+    """Generates a sparse TileDB array and fills it with random float32 data.
+
+    Args:
+        uri:
+            URI for the new TileDB array.
+
+        rows:
+            Number of rows.
+
+        cols:
+            Number of columns (features).
+
+        density:
+            Density of the sparse matrix.
+
+        attr_name:
+            Name of the attribute.
+
+        attr_dtype:
+            Data type of the attribute.
+
+        chunk_size:
+            Number of rows to generate and write per batch.
+
+        tiledb_configs:
+            TileDB context configuration.
+
+        sparse_format_to_write:
+            Scipy sparse format to use for generating chunks ('coo', 'csr', 'csc').
+
+    """
+    if tiledb.array_exists(uri):
+        print(f"Array {uri} already exists. Removing.")
+        shutil.rmtree(uri)
+
+    print(f"Creating sparse array at '{uri}' with shape ({rows}, {cols}), density ~{density}")
+    cfg = CellArrConfig(ctx_config=tiledb_config if tiledb_config else {})
+    create_cellarray(
+        uri=uri,
+        shape=(rows, cols),
+        attr_dtype=attr_dtype,
+        sparse=True,
+        dim_names=["rows", "cols"],
+        attr_name=attr_name,
+        # config=cfg
+    )
+
+    ctx = tiledb.Ctx(cfg.ctx_config) if cfg.ctx_config else None
+    arr_writer = SparseCellArray(
+        uri=uri,
+        attr=attr_name,
+        mode="w",
+        config_or_context=ctx,
+    )
+
+    print(f"Writing data to sparse array '{uri}'...")
+    for i in range(0, rows, chunk_size):
+        end_row = min(i + chunk_size, rows)
+        num_chunk_rows = end_row - i
+        if num_chunk_rows <= 0:
+            continue
+
+        data_chunk_scipy = sp.random(
+            num_chunk_rows, cols, density=density, format=sparse_format_to_write, dtype=attr_dtype
+        )
+
+        if data_chunk_scipy.nnz > 0:
+            arr_writer.write_batch(data_chunk_scipy, start_row=i)
+
+        if (i // chunk_size) % 10 == 0:
+            print(f"  Sparse write: {end_row}/{rows} rows processed for writing.")
+
+    print(f"Finished writing to sparse array '{uri}'.")

{cellarr_array-0.0.3.dist-info → cellarr_array-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellarr-array
-Version: 0.0.3
+Version: 0.2.0
 Summary: Base class for handling TileDB backed arrays.
 Home-page: https://github.com/cellarr/cellarr-array
 Author: Jayaram Kancherla
@@ -16,10 +16,13 @@ Requires-Dist: importlib-metadata; python_version < "3.8"
 Requires-Dist: tiledb
 Requires-Dist: numpy
 Requires-Dist: scipy
+Provides-Extra: optional
+Requires-Dist: torch; extra == "optional"
 Provides-Extra: testing
 Requires-Dist: setuptools; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"
+Requires-Dist: torch; extra == "testing"
 Dynamic: license-file
 
 [![PyPI-Server](https://img.shields.io/pypi/v/cellarr-array.svg)](https://pypi.org/project/cellarr-array/)

cellarr_array-0.2.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+cellarr_array/__init__.py,sha256=coBnoCq1_cv6FnnbowNt6wEIDfVl2GlGTkjnveP-8C4,707
+cellarr_array/core/__init__.py,sha256=fvM-FEiDn8TKDbHxhhzp9FXZFNovFwvIUSY6SpLQRdk,98
+cellarr_array/core/base.py,sha256=3FlhzZSh4ePz3Zm_dU8XNXJ6xgs7rKGi5HgCVWJLhXY,13458
+cellarr_array/core/dense.py,sha256=LODRH4utpKs8xhT79Q2-nRiam_s68_a0qPj0unEM7rg,3940
+cellarr_array/core/helpers.py,sha256=Z_2zRUULFTm7Lo9EpkGvIeRraP6XNDRB-o3rh9ChKQQ,7856
+cellarr_array/core/sparse.py,sha256=XifIWhbTRAQ6qL096th-dCkqscNRwFZuTd7uaRf9aGM,8844
+cellarr_array/dataloaders/__init__.py,sha256=U-MfwC2K84OIXT75in41fe_wvoxjUC5Krb5zICQn_O8,245
+cellarr_array/dataloaders/denseloader.py,sha256=JYJlbuX5My64iIPW_-nlPFkNIezxL3Z3mkwInS3hH9M,7291
+cellarr_array/dataloaders/iterabledataloader.py,sha256=lR2T1YatyBlDM5Sy_75B7_8ORiWfn3cp4q48Oujwf-c,11916
+cellarr_array/dataloaders/sparseloader.py,sha256=V_eKw-Z_CNxHP8c2BN3sOuuv6RPiWBzRfW1BYLhNaQc,7962
+cellarr_array/dataloaders/utils.py,sha256=buJ87x1YBTt5-nZoy_I5j6ko1lVlHdiGpQCusdLoRLI,600
+cellarr_array/utils/__init__.py,sha256=DM5jeUMbxbRzTu2QCjpLlrTQ5uionF887S_7i6_952U,177
+cellarr_array/utils/config.py,sha256=67zBxpYY9N_v6TMdyljUIZmckbwOBcuLC99aJooGmfA,2917
+cellarr_array/utils/mock.py,sha256=7GyCbtM7u94pm7qhjsPRSO2IWYLmd4UrjyvLnQtMMkc,4579
+cellarr_array-0.2.0.dist-info/licenses/LICENSE.txt,sha256=JUlHIfWcRe_MZop18pQvMIPLKSSPz3XQ06ASHuW5Wh8,1076
+cellarr_array-0.2.0.dist-info/METADATA,sha256=NbNM3Gyu4t2f1odp26QiUutfic0IdnagSCnJUn9NLSs,4228
+cellarr_array-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cellarr_array-0.2.0.dist-info/top_level.txt,sha256=oErp0D8ABZV-QPtTiXT8_F2z36Ic7ykuDg_1Y84HLZM,14
+cellarr_array-0.2.0.dist-info/RECORD,,

{cellarr_array-0.0.3.dist-info → cellarr_array-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{cellarr_array-0.0.3.dist-info → cellarr_array-0.2.0.dist-info}/licenses/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2025 Jayaram Kancherla
+Copyright (c) 2025 Genentech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

cellarr_array/CellArray.py

@@ -1,251 +0,0 @@
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-
-try:
-    from types import EllipsisType
-except ImportError:
-    # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025
-    EllipsisType = type(...)
-from typing import List, Literal, Optional, Tuple, Union
-
-import numpy as np
-import tiledb
-from scipy import sparse
-
-from .config import ConsolidationConfig
-from .helpers import SliceHelper
-
-__author__ = "Jayaram Kancherla"
-__copyright__ = "Jayaram Kancherla"
-__license__ = "MIT"
-
-
-class CellArray(ABC):
-    """Abstract base class for TileDB array operations."""
-
-    def __init__(
-        self,
-        uri: str,
-        attr: str = "data",
-        mode: Optional[Literal["r", "w", "n", "d"]] = None,
-        config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None,
-        validate: bool = True,
-    ):
-        """Initialize the object.
-
-        Args:
-            uri:
-                URI to the array.
-
-            attr:
-                Attribute to access.
-                Defaults to "data".
-
-            mode:
-                Open the array object in read 'r', write 'w', modify
-                exclusive 'm' mode, or delete 'd' mode.
-
-                Defaults to None for automatic mode switching.
-
-            config_or_context:
-                Optional config or context object.
-
-                Defaults to None.
-
-            validate:
-                Whether to validate the attributes.
-                Defaults to True.
-        """
-        self.uri = uri
-        self._mode = mode
-
-        if config_or_context is None:
-            # config_or_context = tiledb.Config()
-            ctx = None
-        else:
-            if isinstance(config_or_context, tiledb.Config):
-                ctx = tiledb.Ctx(config_or_context)
-            elif isinstance(config_or_context, tiledb.Ctx):
-                ctx = config_or_context
-            else:
-                raise TypeError("'config_or_context' must be either TileDB config or a context object.")
-
-        self._ctx = ctx
-        self._array = None
-        self._shape = None
-        self._ndim = None
-        self._dim_names = None
-        self._attr_names = None
-        self._nonempty_domain = None
-
-        if validate:
-            self._validate(attr=attr)
-
-        self._attr = attr
-
-    def _validate(self, attr):
-        with self.open_array(mode="r") as A:
-            if A.ndim > 2:
-                raise ValueError("Only 1D and 2D arrays are supported.")
-
-            if attr not in self.attr_names:
-                raise ValueError(
-                    f"Attribute '{attr}' does not exist in the array. Available attributes: {self.attr_names}."
-                )
-
-    @property
-    def mode(self) -> Optional[str]:
-        """Get current array mode."""
-        return self._mode
-
-    @mode.setter
-    def mode(self, value: Optional[str]):
-        """Set array mode.
-
-        Args:
-            value:
-                One of `None`, 'r', 'w', or 'm', 'd'.
-        """
-        if value is not None and value not in ["r", "w", "m", "d"]:
-            raise ValueError("Mode must be one of: None, 'r', 'w', 'm', 'd'")
-        self._mode = value
-
-    @property
-    def dim_names(self) -> List[str]:
-        """Get dimension names of the array."""
-        if self._dim_names is None:
-            with self.open_array(mode="r") as A:
-                self._dim_names = [dim.name for dim in A.schema.domain]
-        return self._dim_names
-
-    @property
-    def attr_names(self) -> List[str]:
-        """Get attribute names of the array."""
-        if self._attr_names is None:
-            with self.open_array(mode="r") as A:
-                self._attr_names = [A.schema.attr(i).name for i in range(A.schema.nattr)]
-        return self._attr_names
-
-    @property
-    def shape(self) -> Tuple[int, ...]:
-        """Get array shape from schema domain."""
-        if self._shape is None:
-            with self.open_array(mode="r") as A:
-                self._shape = tuple(int(dim.domain[1] - dim.domain[0] + 1) for dim in A.schema.domain)
-        return self._shape
-
-    @property
-    def nonempty_domain(self) -> Tuple[int, ...]:
-        """Get array non-empty domain."""
-        if self._nonempty_domain is None:
-            with self.open_array(mode="r") as A:
-                self._nonempty_domain = A.nonempty_domain()
-        return self._nonempty_domain
-
-    @property
-    def ndim(self) -> int:
-        """Get number of dimensions."""
-        if self._ndim is None:
-            self._ndim = len(self.shape)
-        return self._ndim
-
-    @contextmanager
-    def open_array(self, mode: Optional[str] = None):
-        """Context manager for array operations.
-
-        Args:
-            mode:
-                Override mode for this operation.
-        """
-        mode = mode if mode is not None else self.mode
-        mode = mode if mode is not None else "r"  # Default to read mode
-
-        array = tiledb.open(self.uri, mode=mode, ctx=self._ctx)
-        try:
-            yield array
-        finally:
-            array.close()
-
-    def __getitem__(self, key: Union[slice, EllipsisType, Tuple[Union[slice, List[int]], ...], EllipsisType]):
-        """Get item implementation that routes to either direct slicing or multi_index
-        based on the type of indices provided.
-
-        Args:
-            key:
-                Slice or list of indices for each dimension in the array.
-        """
-        if not isinstance(key, tuple):
-            key = (key,)
-
-        if len(key) > self.ndim:
-            raise IndexError(f"Invalid number of dimensions: got {len(key)}, expected {self.ndim}")
-
-        # Normalize all indices
-        normalized_key = tuple(SliceHelper.normalize_index(idx, self.shape[i]) for i, idx in enumerate(key))
-
-        num_ellipsis = sum(isinstance(i, EllipsisType) for i in normalized_key)
-        if num_ellipsis > 1:
-            raise IndexError(f"Found more than 1 Ellipsis (...) in key: {normalized_key}")
-
-        # Check if we can use direct slicing
-        use_direct = all(isinstance(idx, (slice, EllipsisType)) for idx in normalized_key)
-
-        if use_direct:
-            return self._direct_slice(normalized_key)
-        else:
-            if num_ellipsis > 0:
-                raise IndexError(f"tiledb does not support ellipsis in multi-index access: {normalized_key}")
-            return self._multi_index(normalized_key)
-
-    @abstractmethod
-    def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarray:
-        """Implementation for direct slicing."""
-        pass
-
-    @abstractmethod
-    def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray:
-        """Implementation for multi-index access."""
-        pass
-
-    def vacuum(self) -> None:
-        """Remove deleted fragments from the array."""
-        tiledb.vacuum(self.uri)
-
-    def consolidate(self, config: Optional[ConsolidationConfig] = None) -> None:
-        """Consolidate array fragments.
-
-        Args:
-            config:
-                Optional consolidation configuration.
-        """
-        if config is None:
-            config = ConsolidationConfig()
-
-        consolidation_cfg = tiledb.Config()
-
-        consolidation_cfg["sm.consolidation.steps"] = config.steps
-        consolidation_cfg["sm.consolidation.step_min_frags"] = config.step_min_frags
-        consolidation_cfg["sm.consolidation.step_max_frags"] = config.step_max_frags
-        consolidation_cfg["sm.consolidation.buffer_size"] = config.buffer_size
-        consolidation_cfg["sm.mem.total_budget"] = config.total_budget
-
-        tiledb.consolidate(self.uri, config=consolidation_cfg)
-
-        if config.vacuum_after:
-            self.vacuum()
-
-    @abstractmethod
-    def write_batch(self, data: Union[np.ndarray, sparse.spmatrix], start_row: int, **kwargs) -> None:
-        """Write a batch of data to the array starting at the specified row.
-
-        Args:
-            data:
-                Data to write (numpy array for dense, scipy sparse matrix for sparse).
-
-            start_row:
-                Starting row index for writing.
-
-            **kwargs:
-                Additional arguments for write operation.
-        """
-        pass

cellarr_array-0.0.3.dist-info/RECORD

@@ -1,11 +0,0 @@
-cellarr_array/CellArray.py,sha256=vc_6oDLCpVgUaP8HsQz4vE0ZyJ1SPdX43s7VQyh7gF0,8204
-cellarr_array/DenseCellArray.py,sha256=rlu2xq8SONwIswqe0TzRNCwM5f0HYgxr4QBtvbBe8ro,3953
-cellarr_array/SparseCellArray.py,sha256=cOIbs_97j5u13FU7FfEfRNqAZi8rHUkypgLgRcubXrU,7304
-cellarr_array/__init__.py,sha256=IUE9wMDISgRkWp-Fc0KJpDiezCJ61kzuTqS9HdK-JeE,779
-cellarr_array/config.py,sha256=67zBxpYY9N_v6TMdyljUIZmckbwOBcuLC99aJooGmfA,2917
-cellarr_array/helpers.py,sha256=ZqK_josEzKzTMP62P9pb4qBiOTisFofTCnu1LETYJT4,6449
-cellarr_array-0.0.3.dist-info/licenses/LICENSE.txt,sha256=qI2hRZobcUlj8gqFqXwqt522HeYyWvHLF00zCSZofHA,1084
-cellarr_array-0.0.3.dist-info/METADATA,sha256=1KgSZEEF2i9aCr4mkkyAmuPXht4y5ZG2-YdH4dBELpQ,4120
-cellarr_array-0.0.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-cellarr_array-0.0.3.dist-info/top_level.txt,sha256=oErp0D8ABZV-QPtTiXT8_F2z36Ic7ykuDg_1Y84HLZM,14
-cellarr_array-0.0.3.dist-info/RECORD,,