lamindb 0.69.9__py3-none-any.whl → 0.70.0__py3-none-any.whl

lamindb/core/_mapped_collection.py

@@ -11,7 +11,9 @@ from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
 
 from .storage._backed_access import (
+    ArrayType,
     ArrayTypes,
+    GroupType,
     GroupTypes,
     StorageType,
     _safer_read_index,
@@ -55,6 +57,12 @@ class MappedCollection:
     If your `AnnData` collection is in the cloud, move them into a local cache
     first for faster access.
 
+    `__getitem__` of the `MappedCollection` object takes a single integer index
+    and returns a dictionary with the observation data sample for this index from
+    the `AnnData` objects in `path_list`. The dictionary has keys for `layers_keys`
+    (`.X` is in `"X"`), `obs_keys`, `obsm_keys` (under `f"obsm_{key}"`) and also `"_store_idx"`
+    for the index of the `AnnData` object containing this observation sample.
+
     .. note::
 
         For a guide, see :doc:`docs:scrna5`.
@@ -70,23 +78,28 @@ class MappedCollection:
 
     Args:
         path_list: A list of paths to `AnnData` objects stored in `.h5ad` or `.zarr` formats.
-        label_keys: Columns of the ``.obs`` slot that store labels.
+        layers_keys: Keys from the ``.layers`` slot. ``layers_keys=None`` or ``"X"`` in the list
+            retrieves ``.X``.
+        obsm_keys: Keys from the ``.obsm`` slots.
+        obs_keys: Keys from the ``.obs`` slots.
         join: `"inner"` or `"outer"` virtual joins. If ``None`` is passed,
             does not join.
         encode_labels: Encode labels into integers.
-            Can be a list with elements from ``label_keys```.
+            Can be a list with elements from ``obs_keys``.
         unknown_label: Encode this label to -1.
-            Can be a dictionary with keys from ``label_keys`` if ``encode_labels=True```
+            Can be a dictionary with keys from ``obs_keys`` if ``encode_labels=True``
             or from ``encode_labels`` if it is a list.
-        cache_categories: Enable caching categories of ``label_keys`` for faster access.
+        cache_categories: Enable caching categories of ``obs_keys`` for faster access.
         parallel: Enable sampling with multiple processes.
-        dtype: Convert numpy arrays from ``.X`` to this dtype on selection.
+        dtype: Convert numpy arrays from ``.X``, ``.layers`` and ``.obsm``
     """
 
     def __init__(
         self,
         path_list: list[UPathStr],
-        label_keys: str | list[str] | None = None,
+        layers_keys: str | list[str] | None = None,
+        obs_keys: str | list[str] | None = None,
+        obsm_keys: str | list[str] | None = None,
         join: Literal["inner", "outer"] | None = "inner",
         encode_labels: bool | list[str] = True,
         unknown_label: str | dict[str, str] | None = None,
@@ -96,27 +109,37 @@ class MappedCollection:
     ):
         assert join in {None, "inner", "outer"}
 
-        label_keys = [label_keys] if isinstance(label_keys, str) else label_keys
-        self.label_keys = label_keys
+        if layers_keys is None:
+            self.layers_keys = ["X"]
+        else:
+            self.layers_keys = (
+                [layers_keys] if isinstance(layers_keys, str) else layers_keys
+            )
+
+        obsm_keys = [obsm_keys] if isinstance(obsm_keys, str) else obsm_keys
+        self.obsm_keys = obsm_keys
+
+        obs_keys = [obs_keys] if isinstance(obs_keys, str) else obs_keys
+        self.obs_keys = obs_keys
 
         if isinstance(encode_labels, list):
             if len(encode_labels) == 0:
                 encode_labels = False
-            elif label_keys is None or not all(
-                enc_label in label_keys for enc_label in encode_labels
+            elif obs_keys is None or not all(
+                enc_label in obs_keys for enc_label in encode_labels
             ):
                 raise ValueError(
-                    "All elements of `encode_labels` should be in `label_keys`."
+                    "All elements of `encode_labels` should be in `obs_keys`."
                 )
         else:
             if encode_labels:
-                encode_labels = label_keys if label_keys is not None else False
+                encode_labels = obs_keys if obs_keys is not None else False
         self.encode_labels = encode_labels
 
         if encode_labels and isinstance(unknown_label, dict):
             if not all(unkey in encode_labels for unkey in unknown_label):  # type: ignore
                 raise ValueError(
-                    "All keys of `unknown_label` should be in `encode_labels` and `label_keys`."
+                    "All keys of `unknown_label` should be in `encode_labels` and `obs_keys`."
                 )
         self.unknown_label = unknown_label
 
@@ -141,12 +164,16 @@ class MappedCollection:
 
         self.join_vars = join
         self.var_indices = None
+        self.var_joint = None
+        self.n_vars_list = None
+        self.n_vars = None
         if self.join_vars is not None:
             self._make_join_vars()
+            self.n_vars = len(self.var_joint)
 
-        if self.label_keys is not None:
+        if self.obs_keys is not None:
             if cache_categories:
-                self._cache_categories(self.label_keys)
+                self._cache_categories(self.obs_keys)
             else:
                 self._cache_cats: dict = {}
             self.encoders: dict = {}
@@ -169,10 +196,10 @@ class MappedCollection:
             self.conns.append(conn)
             self.storages.append(storage)
 
-    def _cache_categories(self, label_keys: list):
+    def _cache_categories(self, obs_keys: list):
         self._cache_cats = {}
         decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
-        for label in label_keys:
+        for label in obs_keys:
             self._cache_cats[label] = []
             for storage in self.storages:
                 with _Connect(storage) as store:
@@ -197,11 +224,13 @@ class MappedCollection:
 
     def _make_join_vars(self):
         var_list = []
+        self.n_vars_list = []
         for storage in self.storages:
             with _Connect(storage) as store:
-                var_list.append(_safer_read_index(store["var"]))
+                vars = _safer_read_index(store["var"])
+                var_list.append(vars)
+                self.n_vars_list.append(len(vars))
 
-        self.var_joint = None
         vars_eq = all(var_list[0].equals(vrs) for vrs in var_list[1:])
         if vars_eq:
             self.join_vars = None
@@ -223,6 +252,20 @@ class MappedCollection:
     def __len__(self):
         return self.n_obs
 
+    @property
+    def shape(self):
+        """Shape of the (virtually aligned) dataset."""
+        return (self.n_obs, self.n_vars)
+
+    @property
+    def original_shapes(self):
+        """Shapes of the underlying AnnData objects."""
+        if self.n_vars_list is None:
+            n_vars_list = [None] * len(self.n_obs_list)
+        else:
+            n_vars_list = self.n_vars_list
+        return list(zip(self.n_obs_list, n_vars_list))
+
     def __getitem__(self, idx: int):
         obs_idx = self.indices[idx]
         storage_idx = self.storage_idx[idx]
@@ -232,17 +275,28 @@ class MappedCollection:
             var_idxs_join = None
 
         with _Connect(self.storages[storage_idx]) as store:
-            out = {"x": self._get_data_idx(store, obs_idx, var_idxs_join)}
-            out["_storage_idx"] = storage_idx
-            if self.label_keys is not None:
-                for label in self.label_keys:
+            out = {}
+            for layers_key in self.layers_keys:
+                lazy_data = (
+                    store["X"] if layers_key == "X" else store["layers"][layers_key]
+                )
+                out[layers_key] = self._get_data_idx(
+                    lazy_data, obs_idx, self.join_vars, var_idxs_join, self.n_vars
+                )
+            if self.obsm_keys is not None:
+                for obsm_key in self.obsm_keys:
+                    lazy_data = store["obsm"][obsm_key]
+                    out[f"obsm_{obsm_key}"] = self._get_data_idx(lazy_data, obs_idx)
+            out["_store_idx"] = storage_idx
+            if self.obs_keys is not None:
+                for label in self.obs_keys:
                     if label in self._cache_cats:
                         cats = self._cache_cats[label][storage_idx]
                         if cats is None:
                             cats = []
                     else:
                         cats = None
-                    label_idx = self._get_label_idx(store, obs_idx, label, cats)
+                    label_idx = self._get_obs_idx(store, obs_idx, label, cats)
                     if label in self.encoders:
                         label_idx = self.encoders[label][label_idx]
                     out[label] = label_idx
@@ -250,46 +304,46 @@ class MappedCollection:
 
     def _get_data_idx(
         self,
-        storage: StorageType,  # type: ignore
+        lazy_data: ArrayType | GroupType,  # type: ignore
         idx: int,
+        join_vars: Literal["inner", "outer"] | None = None,
         var_idxs_join: list | None = None,
-        layer_key: str | None = None,
+        n_vars_out: int | None = None,
     ):
         """Get the index for the data."""
-        layer = storage["X"] if layer_key is None else storage["layers"][layer_key]  # type: ignore
-        if isinstance(layer, ArrayTypes):  # type: ignore
-            layer_idx = layer[idx]
-            if self.join_vars is None:
-                result = layer_idx
+        if isinstance(lazy_data, ArrayTypes):  # type: ignore
+            lazy_data_idx = lazy_data[idx]  # type: ignore
+            if join_vars is None:
+                result = lazy_data_idx
                 if self._dtype is not None:
                     result = result.astype(self._dtype, copy=False)
-            elif self.join_vars == "outer":
-                dtype = layer_idx.dtype if self._dtype is None else self._dtype
-                result = np.zeros(len(self.var_joint), dtype=dtype)
-                result[var_idxs_join] = layer_idx
+            elif join_vars == "outer":
+                dtype = lazy_data_idx.dtype if self._dtype is None else self._dtype
+                result = np.zeros(n_vars_out, dtype=dtype)
+                result[var_idxs_join] = lazy_data_idx
             else:  # inner join
-                result = layer_idx[var_idxs_join]
+                result = lazy_data_idx[var_idxs_join]
                 if self._dtype is not None:
                     result = result.astype(self._dtype, copy=False)
             return result
         else:  # assume csr_matrix here
-            data = layer["data"]
-            indices = layer["indices"]
-            indptr = layer["indptr"]
+            data = lazy_data["data"]  # type: ignore
+            indices = lazy_data["indices"]  # type: ignore
+            indptr = lazy_data["indptr"]  # type: ignore
             s = slice(*(indptr[idx : idx + 2]))
             data_s = data[s]
             dtype = data_s.dtype if self._dtype is None else self._dtype
-            if self.join_vars == "outer":
-                layer_idx = np.zeros(len(self.var_joint), dtype=dtype)
-                layer_idx[var_idxs_join[indices[s]]] = data_s
+            if join_vars == "outer":
+                lazy_data_idx = np.zeros(n_vars_out, dtype=dtype)
+                lazy_data_idx[var_idxs_join[indices[s]]] = data_s
             else:
-                layer_idx = np.zeros(layer.attrs["shape"][1], dtype=dtype)
-                layer_idx[indices[s]] = data_s
-                if self.join_vars == "inner":
-                    layer_idx = layer_idx[var_idxs_join]
-            return layer_idx
+                lazy_data_idx = np.zeros(lazy_data.attrs["shape"][1], dtype=dtype)  # type: ignore
+                lazy_data_idx[indices[s]] = data_s
+                if join_vars == "inner":
+                    lazy_data_idx = lazy_data_idx[var_idxs_join]
+            return lazy_data_idx
 
-    def _get_label_idx(
+    def _get_obs_idx(
         self,
         storage: StorageType,
         idx: int,
@@ -317,12 +371,12 @@ class MappedCollection:
             label = label.decode("utf-8")
         return label
 
-    def get_label_weights(self, label_keys: str | list[str]):
+    def get_label_weights(self, obs_keys: str | list[str]):
         """Get all weights for the given label keys."""
-        if isinstance(label_keys, str):
-            label_keys = [label_keys]
+        if isinstance(obs_keys, str):
+            obs_keys = [obs_keys]
         labels_list = []
-        for label_key in label_keys:
+        for label_key in obs_keys:
             labels_to_str = self.get_merged_labels(label_key).astype(str).astype("O")
             labels_list.append(labels_to_str)
         if len(labels_list) > 1:

lamindb/core/_run_context.py

@@ -360,7 +360,6 @@ class run_context:
                 run=cls.run,
                 transform=cls.transform,
                 filepath=cls.path,
-                is_consecutive=True,
             )
         return None
 

lamindb/core/_settings.py

@@ -64,7 +64,7 @@ class Settings:
     FAQ: :doc:`/faq/idempotency`
     """
     track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.stage()` and `.backed()`.
+    """Track files as input upon `.load()`, `.cache()` and `.backed()`.
 
     Requires a global run context with :func:`~lamindb.track` was created!
 

lamindb/core/datasets/_core.py

@@ -299,7 +299,7 @@ def anndata_human_immune_cells(
         adata = sc.read('Global.h5ad')
         adata.obs = adata.obs[['donor_id', 'tissue', 'cell_type', 'assay', 'tissue_ontology_term_id', 'cell_type_ontology_term_id', 'assay_ontology_term_id']].copy()
         sc.pp.subsample(adata, fraction=0.005)
-        del adata.uns["development_stage_ontology_term_id_colors"]
+        del adata.uns["development_cache_ontology_term_id_colors"]
         del adata.uns["sex_ontology_term_id_colors"]
         adata.write('human_immune.h5ad')
     """
@@ -401,7 +401,47 @@ def mudata_papalexi21_subset():  # pragma: no cover
         "papalexi21_subset.h5mu",
     )
 
-    return md.read_h5mu(filepath)
+    mdata = md.read_h5mu(filepath)
+    for mod in ["rna", "adt", "hto", "gdo"]:
+        mdata[mod].obs.drop(
+            mdata[mod].obs.columns, axis=1, inplace=True
+        )  # Drop all columns
+    for col in mdata.obs.columns:
+        for mod in ["rna", "adt", "hto", "gdo"]:
+            if col.endswith(f"_{mod.upper()}"):
+                new_col = col.replace(f"{mod}:", "")
+                if new_col != col:
+                    mdata[mod].obs[new_col] = mdata.obs.pop(col)
+            else:
+                new_col = col.replace(f"{mod}:", "")
+                if new_col not in mdata.obs.columns and col in mdata.obs.columns:
+                    mdata.obs[new_col] = mdata.obs.pop(col)
+
+    for col in mdata.obs.columns:
+        for mod in ["rna", "adt", "hto", "gdo"]:
+            if col.endswith(f"_{mod.upper()}"):
+                del mdata.obs[col]
+
+    for col in [
+        "orig.ident",
+        "MULTI_ID",
+        "NT",
+        "S.Score",
+        "G2M.Score",
+        "Phase",
+        "gene_target",
+        "guide_ID",
+        "HTO_classification",
+    ]:
+        del mdata.obs[col]
+        mdata.update()
+
+    mdata["rna"].obs["percent.mito"] = mdata.obs.pop("percent.mito")
+    mdata["hto"].obs["technique"] = "cell hashing"
+    mdata["hto"].obs["technique"] = mdata["hto"].obs["technique"].astype("category")
+    mdata.update()
+
+    return mdata
 
 
 def df_iris() -> pd.DataFrame:

lamindb/core/storage/_backed_access.py

@@ -100,7 +100,7 @@ def _records_to_df(obj):
         return obj
 
 
-class Registry:
+class AccessRegistry:
     def __init__(self):
         self._registry = {}
         self._openers = {}
@@ -141,7 +141,7 @@ class Registry:
 
 
 # storage specific functions should be registered and called through the registry
-registry = Registry()
+registry = AccessRegistry()
 
 
 @registry.register_open("h5py")
@@ -176,8 +176,12 @@ def safer_read_partial(elem, indices):
         indices_increasing = []
         indices_inverse = []
         for indices_dim in indices:
-            if isinstance(indices_dim, np.ndarray) and not np.all(
-                np.diff(indices_dim) > 0
+            # should be integer or bool
+            # ignore bool or increasing unique integers
+            if (
+                isinstance(indices_dim, np.ndarray)
+                and indices_dim.dtype != "bool"
+                and not np.all(np.diff(indices_dim) > 0)
             ):
                 idx_unique, idx_inverse = np.unique(indices_dim, return_inverse=True)
                 indices_increasing.append(idx_unique)

lamindb/core/storage/file.py

@@ -22,6 +22,7 @@ from lnschema_core.models import Artifact, Storage
 from lamindb.core._settings import settings
 
 if TYPE_CHECKING:
+    import mudata as md
     from lamindb_setup.core.types import UPathStr
 
 try:
@@ -170,6 +171,13 @@ def read_tsv(path: UPathStr, **kwargs) -> pd.DataFrame:
     return pd.read_csv(path_sanitized, sep="\t", **kwargs)
 
 
+def read_mdata_h5mu(filepath: UPathStr, **kwargs) -> md.MuData:
+    import mudata as md
+
+    path_sanitized = Path(filepath)
+    return md.read_h5mu(path_sanitized, **kwargs)
+
+
 def load_html(path: UPathStr):
     if is_run_from_ipython:
         with open(path, encoding="utf-8") as f:
@@ -224,6 +232,7 @@ def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
         ".zrad": read_adata_zarr,
         ".html": load_html,
         ".json": load_json,
+        ".h5mu": read_mdata_h5mu,
     }
 
     reader = READER_FUNCS.get(filepath.suffix)

lamindb/core/storage/object.py

@@ -9,6 +9,14 @@ if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
 
+def _mudata_is_installed():
+    try:
+        import mudata
+    except ImportError:
+        return False
+    return True
+
+
 def infer_suffix(dmem, adata_format: str | None = None):
     """Infer LaminDB storage file suffix from a data object."""
     if isinstance(dmem, AnnData):
@@ -25,6 +33,11 @@ def infer_suffix(dmem, adata_format: str | None = None):
     elif isinstance(dmem, DataFrame):
         return ".parquet"
     else:
+        if _mudata_is_installed():
+            from mudata import MuData
+
+            if isinstance(dmem, MuData):
+                return ".h5mu"
         raise NotImplementedError
 
 
@@ -34,4 +47,10 @@ def write_to_file(dmem, filepath: UPathStr):
     elif isinstance(dmem, DataFrame):
         dmem.to_parquet(filepath)
     else:
+        if _mudata_is_installed():
+            from mudata import MuData
+
+            if isinstance(dmem, MuData):
+                dmem.write(filepath)
+                return
         raise NotImplementedError

lamindb/integrations/_vitessce.py

@@ -2,39 +2,48 @@ from __future__ import annotations
 
 import json
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 
 from lamindb._artifact import Artifact
 
+if TYPE_CHECKING:
+    from vitessce import VitessceConfig
+
 
-# tested in lamin-spatial
-# can't type vitessce_config because can't assume it's installed
-def save_vitessce_config(vitessce_config, description: str) -> Artifact:
+# tested & context in https://github.com/laminlabs/lamin-spatial
+def save_vitessce_config(vitessce_config: VitessceConfig, description: str) -> Artifact:
     """Takes a ``VitessceConfig`` object and saves it as an artifact.
 
     Args:
         vitessce_config (``VitessceConfig``): A VitessceConfig object.
         description: A description for the artifact.
     """
+    # can't assume vitessce is installed
     from vitessce import VitessceConfig
 
-    assert isinstance(vitessce_config, VitessceConfig)
+    # create a local _data export_ in a folder
     timestamp = datetime.now(timezone.utc).isoformat().split(".")[0]
-    vitesse_export = f"./vitessce_export_{timestamp}.vitessce"
-    vitessce_config.export(to="files", base_url="", out_dir=vitesse_export)
-    logger.important(f"local export: {vitesse_export}")
-    artifact = Artifact(vitesse_export, description=description)
+    vitesse_export_folder = f"./vitessce_export_{timestamp}.vitessce"
+    vitessce_config.export(to="files", base_url="", out_dir=vitesse_export_folder)
+    logger.important(f"local export: {vitesse_export_folder}")
+    # create an artifact and store the local export in th cloud
+    artifact = Artifact(vitesse_export_folder, description=description)
     artifact.save()
+    # create a JSON export that points to the data in the cloud
     config_dict = vitessce_config.to_dict(base_url=artifact.path.to_url())
     logger.important(f"base url: {artifact.path.to_url()}")
+    # manually place that JSON export into the local data export folder
     config_filename = "vitessce_config.json"
-    config_file_local_path = f"{vitesse_export}/{config_filename}"
+    config_file_local_path = f"{vitesse_export_folder}/{config_filename}"
     with open(config_file_local_path, "w") as file:
         json.dump(config_dict, file)
+    # manually place that JSON export into the previously registered artifact folder
     config_file_path = artifact.path / config_filename
     config_file_path.upload_from(config_file_local_path)
+    # log the the URLs
     logger.important(f"config url: {config_file_path.to_url()}")
     slug = ln_setup.settings.instance.slug
     logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")

{lamindb-0.69.9.dist-info → lamindb-0.70.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.9
+Version: 0.70.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,15 +9,15 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.64.6
-Requires-Dist: lamindb_setup==0.69.0
+Requires-Dist: lnschema_core==0.65.0
+Requires-Dist: lamindb_setup==0.69.2
 Requires-Dist: lamin_utils==0.13.1
-Requires-Dist: lamin_cli==0.12.1
+Requires-Dist: lamin_cli==0.12.3
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<0.10.6
+Requires-Dist: anndata>=0.8.0,<0.10.8
 Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
 Requires-Dist: pandas
@@ -27,7 +27,7 @@ Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
 Requires-Dist: s3fs==2023.12.2 ; extra == "aws"
 Requires-Dist: fsspec[s3]==2023.12.2 ; extra == "aws"
-Requires-Dist: bionty==0.42.5 ; extra == "bionty"
+Requires-Dist: bionty==0.42.8 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -37,7 +37,6 @@ Requires-Dist: coverage ; extra == "dev"
 Requires-Dist: pytest-cov ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
-Requires-Dist: lamin_vault ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.7 ; extra == "fcs"
 Requires-Dist: fsspec[gs]==2023.12.2 ; extra == "gcp"
@@ -65,7 +64,7 @@ Provides-Extra: zarr
 - Track data lineage across notebooks & pipelines.
 - Integrate registries for experimental metadata & in-house ontologies.
 - Validate, standardize & annotate.
-- Collaborate across distributed LaminDB instances.
+- Collaborate across distributed databases.
 
 ## Documentation