lamindb 0.76.0__py3-none-any.whl → 0.76.2__py3-none-any.whl

lamindb/core/_mapped_collection.py

@@ -149,7 +149,7 @@ class MappedCollection:
         self.storages = []  # type: ignore
         self.conns = []  # type: ignore
         self.parallel = parallel
-        self._path_list = path_list
+        self.path_list = path_list
         self._make_connections(path_list, parallel)
 
         self.n_obs_list = []
@@ -165,11 +165,12 @@ class MappedCollection:
         self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
         self.storage_idx = np.repeat(np.arange(len(self.storages)), self.n_obs_list)
 
-        self.join_vars = join
-        self.var_indices = None
-        self.var_joint = None
-        self.n_vars_list = None
-        self.n_vars = None
+        self.join_vars: Literal["inner", "outer"] | None = join
+        self.var_indices: list | None = None
+        self.var_joint: pd.Index | None = None
+        self.n_vars_list: list | None = None
+        self.var_list: list | None = None
+        self.n_vars: int | None = None
         if self.join_vars is not None:
             self._make_join_vars()
             self.n_vars = len(self.var_joint)
@@ -225,43 +226,71 @@ class MappedCollection:
             encoder.update({cat: i for i, cat in enumerate(cats)})
             self.encoders[label] = encoder
 
-    def _make_join_vars(self):
-        var_list = []
+    def _read_vars(self):
+        self.var_list = []
         self.n_vars_list = []
         for storage in self.storages:
             with _Connect(storage) as store:
                 vars = _safer_read_index(store["var"])
-                var_list.append(vars)
+                self.var_list.append(vars)
                 self.n_vars_list.append(len(vars))
 
-        vars_eq = all(var_list[0].equals(vrs) for vrs in var_list[1:])
+    def _make_join_vars(self):
+        if self.var_list is None:
+            self._read_vars()
+        vars_eq = all(self.var_list[0].equals(vrs) for vrs in self.var_list[1:])
         if vars_eq:
             self.join_vars = None
-            self.var_joint = var_list[0]
+            self.var_joint = self.var_list[0]
             return
 
         if self.join_vars == "inner":
-            self.var_joint = reduce(pd.Index.intersection, var_list)
+            self.var_joint = reduce(pd.Index.intersection, self.var_list)
             if len(self.var_joint) == 0:
                 raise ValueError(
                     "The provided AnnData objects don't have shared varibales.\n"
                     "Use join='outer'."
                 )
-            self.var_indices = [vrs.get_indexer(self.var_joint) for vrs in var_list]
+            self.var_indices = [
+                vrs.get_indexer(self.var_joint) for vrs in self.var_list
+            ]
         elif self.join_vars == "outer":
-            self.var_joint = reduce(pd.Index.union, var_list)
-            self.var_indices = [self.var_joint.get_indexer(vrs) for vrs in var_list]
+            self.var_joint = reduce(pd.Index.union, self.var_list)
+            self.var_indices = [
+                self.var_joint.get_indexer(vrs) for vrs in self.var_list
+            ]
+
+    def check_vars_sorted(self, ascending: bool = True) -> bool:
+        """Returns `True` if all variables are sorted in all objects."""
+        if self.var_list is None:
+            self._read_vars()
+        if ascending:
+            vrs_sort_status = (vrs.is_monotonic_increasing for vrs in self.var_list)
+        else:
+            vrs_sort_status = (vrs.is_monotonic_decreasing for vrs in self.var_list)
+        return all(vrs_sort_status)
+
+    def check_vars_non_aligned(self, vars: pd.Index | list) -> list[int]:
+        """Returns indices of objects with non-aligned variables.
+
+        Args:
+            vars: Check alignment against these variables.
+        """
+        if self.var_list is None:
+            self._read_vars()
+        vars = pd.Index(vars)
+        return [i for i, vrs in enumerate(self.var_list) if not vrs.equals(vars)]
 
     def __len__(self):
         return self.n_obs
 
     @property
-    def shape(self):
+    def shape(self) -> tuple[int, int]:
         """Shape of the (virtually aligned) dataset."""
         return (self.n_obs, self.n_vars)
 
     @property
-    def original_shapes(self):
+    def original_shapes(self) -> list[tuple[int, int]]:
         """Shapes of the underlying AnnData objects."""
         if self.n_vars_list is None:
             n_vars_list = [None] * len(self.n_obs_list)
@@ -374,8 +403,27 @@ class MappedCollection:
             label = label.decode("utf-8")
         return label
 
-    def get_label_weights(self, obs_keys: str | list[str]):
-        """Get all weights for the given label keys."""
+    def get_label_weights(
+        self,
+        obs_keys: str | list[str],
+        scaler: float | None = None,
+        return_categories: bool = False,
+    ):
+        """Get all weights for the given label keys.
+
+        This counts the number of labels for each label and returns
+        weights for each obs label accoding to the formula `1 / num of this label in the data`.
+        If `scaler` is provided, then `scaler / (scaler + num of this label in the data)`.
+
+        Args:
+            obs_keys: A key in the ``.obs`` slots or a list of keys. If a list is provided,
+                the labels from the obs keys will be concatenated with ``"__"`` delimeter
+            scaler: Use this number to scale the provided weights.
+            return_categories: If `False`, returns weights for each observation,
+                can be directly passed to a sampler. If `True`, returns a dictionary with
+                unique categories for labels (concatenated if `obs_keys` is a list)
+                and their weights.
+        """
         if isinstance(obs_keys, str):
             obs_keys = [obs_keys]
         labels_list = []
@@ -383,12 +431,20 @@ class MappedCollection:
             labels_to_str = self.get_merged_labels(label_key).astype(str).astype("O")
             labels_list.append(labels_to_str)
         if len(labels_list) > 1:
-            labels = reduce(lambda a, b: a + b, labels_list)
+            labels = ["__".join(labels_obs) for labels_obs in zip(*labels_list)]
         else:
             labels = labels_list[0]
-        labels = self.get_merged_labels(label_key)
-        counter = Counter(labels)  # type: ignore
-        weights = 1.0 / np.array([counter[label] for label in labels])
+        counter = Counter(labels)
+        if return_categories:
+            return {
+                k: 1.0 / v if scaler is None else scaler / (v + scaler)
+                for k, v in counter.items()
+            }
+        counts = np.array([counter[label] for label in labels])
+        if scaler is None:
+            weights = 1.0 / counts
+        else:
+            weights = scaler / (counts + scaler)
         return weights
 
     def get_merged_labels(self, label_key: str):
@@ -426,7 +482,7 @@ class MappedCollection:
                     codes = self._get_codes(store, label_key)
                     codes = decode(codes) if isinstance(codes[0], bytes) else codes
                     cats_merge.update(codes)
-        return cats_merge
+        return sorted(cats_merge)
 
     def _get_categories(self, storage: StorageType, label_key: str):  # type: ignore
         """Get categories."""
@@ -483,7 +539,7 @@ class MappedCollection:
         self._closed = True
 
     @property
-    def closed(self):
+    def closed(self) -> bool:
         """Check if connections to array streaming backend are closed.
 
         Does not matter if `parallel=True`.
@@ -508,4 +564,4 @@ class MappedCollection:
         mapped.parallel = False
         mapped.storages = []
         mapped.conns = []
-        mapped._make_connections(mapped._path_list, parallel=False)
+        mapped._make_connections(mapped.path_list, parallel=False)

lamindb/core/_settings.py

@@ -54,7 +54,7 @@ class Settings:
     track_run_inputs: bool = True
     """Track files as input upon `.load()`, `.cache()` and `.backed()`.
 
-    Requires a global run context with :func:`~lamindb.track` was created!
+    Requires a global run context with :func:`~lamindb.core.Context.track` was created!
 
     FAQ: :doc:`/faq/track-run-inputs`
     """
@@ -83,14 +83,10 @@ class Settings:
     def transform(self) -> TransformSettings:
         """Transform settings.
 
-        For example::
-
-            ln.settings.transform.stem_uid = "FPnfDtJz8qbE"  # defines version family
-            ln.settings.transform.version = "1"              # defines version
-            ln.settings.transform.name = "My good script"    # semantic name
-
-        The first two are typically auto-generated by :func:`~lamindb.track`.
+        Is deprecated since version 0.76.1.
         """
+        # enable warning soon
+        # logger.warning("Transform settings are deprecated, please instead set `ln.context.uid`")
         return transform_settings
 
     @property

lamindb/core/datasets/_core.py

@@ -432,6 +432,7 @@ def mudata_papalexi21_subset() -> MuData:  # pragma: no cover
     mdata.push_obs(["percent.mito"], mods=["rna"], drop=True)
     mdata["hto"].obs["technique"] = "cell hashing"
     mdata["hto"].obs["technique"] = mdata["hto"].obs["technique"].astype("category")
+    mdata.pull_obs(["technique"], mods="hto")
 
     return mdata
 

lamindb/core/exceptions.py

@@ -5,24 +5,41 @@ The registry base class:
 .. autosummary::
    :toctree: .
 
+   DoesNotExist
    ValidationError
    NotebookNotSavedError
    NoTitleError
-   MissingTransformSettings
-   UpdateTransformSettings
+   MissingContext
+   UpdateContext
    IntegrityError
 
 """
 
 
+class TrackNotCalled(SystemExit):
+    pass
+
+
+class NotebookNotSaved(SystemExit):
+    pass
+
+
 class ValidationError(SystemExit):
     """Validation error: not mapped in registry."""
 
     pass
 
 
+# inspired by Django's DoesNotExist
+# equivalent to SQLAlchemy's NoResultFound
+class DoesNotExist(Exception):
+    """No record found."""
+
+    pass
+
+
 # -------------------------------------------------------------------------------------
-# ln.track() AKA run_context
+# ln.context.track() AKA context
 # -------------------------------------------------------------------------------------
 
 
@@ -48,13 +65,13 @@ class NoTitleError(Exception):
     pass
 
 
-class MissingTransformSettings(SystemExit):
+class MissingContext(SystemExit):
     """User didn't define transform settings."""
 
     pass
 
 
-class UpdateTransformSettings(SystemExit):
+class UpdateContext(SystemExit):
     """Transform settings require update."""
 
     pass

lamindb/core/storage/__init__.py

@@ -18,8 +18,8 @@ Array accessors.
 
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
-from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor
+from ._tiledbsoma import register_for_tiledbsoma_store, write_tiledbsoma_store
 from ._valid_suffixes import VALID_SUFFIXES
 from .objects import infer_suffix, write_to_disk
 from .paths import delete_storage, load_to_memory

lamindb/core/storage/_backed_access.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable, Literal
+from typing import TYPE_CHECKING, Any, Callable
 
 from anndata._io.specs.registry import get_spec
 from lnschema_core import Artifact
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
+from ._tiledbsoma import _open_tiledbsoma
 from .paths import filepath_from_artifact
 
 if TYPE_CHECKING:
@@ -52,43 +53,6 @@ def _track_writes_factory(obj: Any, finalize: Callable):
     return obj
 
 
-def _open_tiledbsoma(
-    filepath: UPath, mode: Literal["r", "w"] = "r"
-) -> SOMACollection | SOMAExperiment:
-    try:
-        import tiledbsoma as soma
-    except ImportError as e:
-        raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
-    filepath_str = filepath.as_posix()
-    if filepath.protocol == "s3":
-        from lamindb_setup.core._settings_storage import get_storage_region
-
-        region = get_storage_region(filepath_str)
-        tiledb_config = {"vfs.s3.region": region}
-        storage_options = filepath.storage_options
-        if "key" in storage_options:
-            tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
-        if "secret" in storage_options:
-            tiledb_config["vfs.s3.aws_secret_access_key"] = storage_options["secret"]
-        if "token" in storage_options:
-            tiledb_config["vfs.s3.aws_session_token"] = storage_options["token"]
-        ctx = soma.SOMATileDBContext(tiledb_config=tiledb_config)
-        # this is a strange bug
-        # for some reason iterdir futher gives incorrect results
-        # if cache is not invalidated
-        # instead of obs and ms it gives ms and ms in the list of names
-        filepath.fs.invalidate_cache()
-    else:
-        ctx = None
-
-    soma_objects = [obj.name for obj in filepath.iterdir()]
-    if "obs" in soma_objects and "ms" in soma_objects:
-        SOMAType = soma.Experiment
-    else:
-        SOMAType = soma.Collection
-    return SOMAType.open(filepath_str, mode=mode, context=ctx)
-
-
 @dataclass
 class BackedAccessor:
     """h5py.File or zarr.Group accessor."""

lamindb/core/storage/_tiledbsoma.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal
+
+from anndata import AnnData
+from lamin_utils import logger
+from lamindb_setup.core._settings_storage import get_storage_region
+from lamindb_setup.core.upath import create_path
+from lnschema_core import Artifact, Run, Storage
+from upath import UPath
+
+if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
+    from tiledbsoma import Collection as SOMACollection
+    from tiledbsoma import Experiment as SOMAExperiment
+    from tiledbsoma.io import ExperimentAmbientLabelMapping
+
+
+def _read_adata_h5ad_zarr(objpath: UPath):
+    from lamindb.core.storage.paths import read_adata_h5ad, read_adata_zarr
+
+    if objpath.is_dir():
+        adata = read_adata_zarr(objpath)
+    else:
+        adata = read_adata_h5ad(objpath)
+    return adata
+
+
+def _tiledb_config_s3(storepath: UPath) -> dict:
+    region = get_storage_region(storepath)
+    tiledb_config = {"vfs.s3.region": region}
+    storage_options = storepath.storage_options
+    if "key" in storage_options:
+        tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
+    if "secret" in storage_options:
+        tiledb_config["vfs.s3.aws_secret_access_key"] = storage_options["secret"]
+    if "token" in storage_options:
+        tiledb_config["vfs.s3.aws_session_token"] = storage_options["token"]
+
+    return tiledb_config
+
+
+def _open_tiledbsoma(
+    storepath: UPath, mode: Literal["r", "w"] = "r"
+) -> SOMACollection | SOMAExperiment:
+    try:
+        import tiledbsoma as soma
+    except ImportError as e:
+        raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
+
+    storepath_str = storepath.as_posix()
+    if storepath.protocol == "s3":
+        ctx = soma.SOMATileDBContext(tiledb_config=_tiledb_config_s3(storepath))
+        # this is a strange bug
+        # for some reason iterdir futher gives incorrect results
+        # if cache is not invalidated
+        # instead of obs and ms it gives ms and ms in the list of names
+        storepath.fs.invalidate_cache()
+    else:
+        ctx = None
+
+    soma_objects = [obj.name for obj in storepath.iterdir()]
+    if "obs" in soma_objects and "ms" in soma_objects:
+        SOMAType = soma.Experiment
+    else:
+        SOMAType = soma.Collection
+    return SOMAType.open(storepath_str, mode=mode, context=ctx)
+
+
+def register_for_tiledbsoma_store(
+    store: UPathStr | Artifact | None,
+    adatas: list[AnnData | UPathStr],
+    measurement_name: str,
+    obs_field_name: str,
+    var_field_name: str,
+    append_obsm_varm: bool = False,
+    run: Run | None = None,
+) -> tuple[ExperimentAmbientLabelMapping, list[AnnData]]:
+    """Register `AnnData` objects to append to `tiledbsoma.Experiment`.
+
+    Pass the returned registration mapping and `AnnData` objects to `write_tiledbsoma_store`.
+
+    See `tiledbsoma.io.from_h5ad
+    <https://tiledbsoma.readthedocs.io/en/latest/_autosummary/tiledbsoma.io.from_h5ad.html>`__.
+    """
+    try:
+        import tiledbsoma as soma
+        import tiledbsoma.io as soma_io
+    except ImportError as e:
+        raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
+
+    if isinstance(store, Artifact):
+        storepath = store.path
+    else:
+        storepath = None if store is None else create_path(store)
+
+    add_run_uid = True
+    ctx = None
+    if storepath is not None:
+        if storepath.protocol == "s3":
+            ctx = soma.SOMATileDBContext(tiledb_config=_tiledb_config_s3(storepath))
+        if storepath.exists():
+            with soma.Experiment.open(
+                storepath.as_posix(), mode="r", context=ctx
+            ) as store:
+                add_run_uid = "lamin_run_uid" in store["obs"].schema.names
+        storepath = storepath.as_posix()
+
+    if add_run_uid:
+        from lamindb.core._data import get_run
+
+        run = get_run(run)
+
+    adata_objects = []
+    for adata in adatas:
+        if isinstance(adata, AnnData):
+            if add_run_uid:
+                if adata.is_view:
+                    raise ValueError(
+                        "Can not register an `AnnData` view, please do `adata.copy()` before passing."
+                    )
+                else:
+                    logger.warning("Mutating in-memory AnnData.")
+                    adata.obs["lamin_run_uid"] = run.uid
+        else:
+            adata = _read_adata_h5ad_zarr(create_path(adata))
+            if add_run_uid:
+                adata.obs["lamin_run_uid"] = run.uid
+        adata_objects.append(adata)
+
+    registration_mapping = soma_io.register_anndatas(
+        experiment_uri=storepath,
+        adatas=adata_objects,
+        measurement_name=measurement_name,
+        obs_field_name=obs_field_name,
+        var_field_name=var_field_name,
+        append_obsm_varm=append_obsm_varm,
+        context=ctx,
+    )
+
+    return registration_mapping, adata_objects
+
+
+def write_tiledbsoma_store(
+    store: Artifact | UPathStr,
+    adata: AnnData | UPathStr,
+    run: Run | None = None,
+    artifact_kwargs: dict | None = None,
+    **kwargs,
+) -> Artifact:
+    """Write `AnnData` to `tiledbsoma.Experiment`.
+
+    Reads `AnnData`, writes it to `tiledbsoma.Experiment` and creates `lamindb.Artifact`.
+
+    See `tiledbsoma.io.from_h5ad
+    <https://tiledbsoma.readthedocs.io/en/latest/_autosummary/tiledbsoma.io.from_h5ad.html>`__.
+    """
+    try:
+        import tiledbsoma as soma
+        import tiledbsoma.io as soma_io
+    except ImportError as e:
+        raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
+
+    from lamindb.core._data import get_run
+
+    if artifact_kwargs is None:
+        artifact_kwargs = {}
+
+    appending: bool = kwargs.get("registration_mapping", None) is not None
+    store_is_artifact: bool = isinstance(store, Artifact)
+    if store_is_artifact:
+        if not appending:
+            raise ValueError(
+                "Trying to append to an existing store without `registration_mapping`."
+            )
+        storepath = store.path
+    else:
+        storepath = create_path(store)
+    add_run_uid: bool = not appending
+
+    if not isinstance(adata, AnnData):
+        # create_path is used
+        # in case adata is somewhere in our managed s3 bucket or just in s3
+        adata = _read_adata_h5ad_zarr(create_path(adata))
+    elif add_run_uid and adata.is_view:
+        raise ValueError(
+            "Can not write from an `AnnData` view, please do `adata.copy()` before passing."
+        )
+
+    run = get_run(run)
+
+    if add_run_uid:
+        adata.obs["lamin_run_uid"] = run.uid
+
+    if storepath.protocol == "s3":
+        ctx = soma.SOMATileDBContext(tiledb_config=_tiledb_config_s3(storepath))
+    else:
+        ctx = None
+
+    soma_io.from_anndata(storepath.as_posix(), adata, context=ctx, **kwargs)
+
+    if add_run_uid:
+        del adata.obs["lamin_run_uid"]
+
+    revises = None
+    if appending:
+        if store_is_artifact:
+            revises = store
+        else:
+            from lamindb._artifact import (
+                check_path_in_existing_storage,
+                get_relative_path_to_directory,
+            )
+
+            storage = check_path_in_existing_storage(storepath)
+            if isinstance(storage, Storage):
+                search_by_key = get_relative_path_to_directory(
+                    path=storepath, directory=UPath(storage.root)
+                ).as_posix()
+                revises = Artifact.filter(
+                    key=search_by_key, is_latest=True, _key_is_virtual=False
+                ).one_or_none()
+                if revises is not None:
+                    logger.info(f"Assuming it is a new version of {revises}.")
+
+    if revises is None:
+        return Artifact(storepath, run=run, **artifact_kwargs)
+    else:
+        return Artifact(storepath, run=run, revises=revises, **artifact_kwargs)

lamindb/core/storage/_valid_suffixes.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from lamindb_setup.core.upath import VALID_COMPOSITE_SUFFIXES, VALID_SIMPLE_SUFFIXES
 
 # add new composite suffixes like so

lamindb/core/storage/paths.py

@@ -84,13 +84,9 @@ def attempt_accessing_path(
             )
     else:
         if artifact._state.db not in ("default", None) and using_key is None:
-            storage = (
-                Storage.using(artifact._state.db).filter(id=artifact.storage_id).one()
-            )
+            storage = Storage.using(artifact._state.db).get(id=artifact.storage_id)
         else:
-            storage = (
-                Storage.objects.using(using_key).filter(id=artifact.storage_id).one()
-            )
+            storage = Storage.objects.using(using_key).get(id=artifact.storage_id)
         # find a better way than passing None to instance_settings in the future!
         storage_settings = StorageSettings(storage.root, access_token=access_token)
     path = storage_settings.key_to_filepath(storage_key)