lamindb 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl

lamindb/_query_set.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Iterable, NamedTuple
 import pandas as pd
 from django.db import models
 from django.db.models import F
+from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import (
     Artifact,
@@ -13,8 +14,10 @@ from lnschema_core.models import (
     Collection,
     IsVersioned,
     Record,
+    Registry,
     Run,
     Transform,
+    VisibilityChoice,
 )
 
 from lamindb.core.exceptions import DoesNotExist
@@ -64,6 +67,55 @@ def one_helper(self):
         return self[0]
 
 
+def process_expressions(registry: Registry, expressions: dict) -> dict:
+    if registry in {Artifact, Collection}:
+        # visibility is set to 0 unless expressions contains id or uid equality
+        if not (
+            "id" in expressions
+            or "uid" in expressions
+            or "uid__startswith" in expressions
+        ):
+            visibility = "visibility"
+            if not any(e.startswith(visibility) for e in expressions):
+                expressions[visibility] = (
+                    VisibilityChoice.default.value
+                )  # default visibility
+            # if visibility is None, do not apply a filter
+            # otherwise, it would mean filtering for NULL values, which doesn't make
+            # sense for a non-NULLABLE column
+            elif visibility in expressions and expressions[visibility] is None:
+                expressions.pop(visibility)
+    return expressions
+
+
+def get(
+    registry_or_queryset: type[Record] | QuerySet,
+    idlike: int | str | None = None,
+    **expressions,
+) -> Record:
+    if isinstance(registry_or_queryset, QuerySet):
+        qs = registry_or_queryset
+        registry = qs.model
+    else:
+        qs = QuerySet(model=registry_or_queryset)
+        registry = registry_or_queryset
+    if isinstance(idlike, int):
+        return super(QuerySet, qs).get(id=idlike)
+    elif isinstance(idlike, str):
+        qs = qs.filter(uid__startswith=idlike)
+        if issubclass(registry, IsVersioned):
+            if len(idlike) <= registry._len_stem_uid:
+                return qs.latest_version().one()
+            else:
+                return qs.one()
+        else:
+            return qs.one()
+    else:
+        assert idlike is None  # noqa: S101
+        expressions = process_expressions(registry, expressions)
+        return registry.objects.get(**expressions)
+
+
 class RecordsList(UserList):
     """Is ordered, can't be queried, but has `.df()`."""
 
@@ -193,6 +245,7 @@ class QuerySet(models.QuerySet, CanValidate):
         # both Transform & Run might reference artifacts
         if self.model in {Artifact, Collection, Transform, Run}:
             for record in self:
+                logger.important(f"deleting {record}")
                 record.delete(*args, **kwargs)
         else:
             self._delete_base_class(*args, **kwargs)
@@ -219,12 +272,12 @@ class QuerySet(models.QuerySet, CanValidate):
             return None
         return self[0]
 
-    def one(self) -> Record:
-        """Exactly one result. Raises error if there are more or none.
+    def get(self, idlike: int | str | None = None, **expressions) -> Record:
+        """Query a single record. Raises error if there are more or none."""
+        return get(self, idlike, **expressions)
 
-        Examples:
-            >>> ln.ULabel.filter(name="benchmark").one()
-        """
+    def one(self) -> Record:
+        """Exactly one result. Raises error if there are more or none."""
         return one_helper(self)
 
     def one_or_none(self) -> Record | None:

lamindb/_record.py

@@ -12,7 +12,7 @@ from lamin_utils._lookup import Lookup
 from lamindb_setup._connect_instance import get_owner_name_from_identifier
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core._hub_core import connect_instance
-from lnschema_core.models import IsVersioned, Record
+from lnschema_core.models import Collection, IsVersioned, Record
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._settings import settings
@@ -90,13 +90,22 @@ def __init__(record: Record, *args, **kwargs):
             match = suggest_records_with_similar_names(record, kwargs)
             if match:
                 if "version" in kwargs:
-                    version_comment = " and version"
-                    existing_record = record.__class__.filter(
-                        name=kwargs["name"], version=kwargs["version"]
-                    ).one_or_none()
+                    if kwargs["version"] is not None:
+                        version_comment = " and version"
+                        existing_record = record.__class__.filter(
+                            name=kwargs["name"], version=kwargs["version"]
+                        ).one_or_none()
+                    else:
+                        # for a versioned record, an exact name match is not a
+                        # criterion for retrieving a record in case `version`
+                        # isn't passed - we'd always pull out many records with exactly the
+                        # same name
+                        existing_record = None
                 else:
                     version_comment = ""
-                    existing_record = record.__class__.filter(name=kwargs["name"]).one()
+                    existing_record = record.__class__.filter(
+                        name=kwargs["name"]
+                    ).one_or_none()
                 if existing_record is not None:
                     logger.important(
                         f"returning existing {record.__class__.__name__} record with same"
@@ -129,23 +138,11 @@ def get(
     **expressions,
 ) -> Record:
     """{}"""  # noqa: D415
-    from lamindb._filter import filter
+    # this is the only place in which we need the lamindb queryset
+    # in this file; everywhere else it should be Django's
+    from lamindb._query_set import QuerySet
 
-    if isinstance(idlike, int):
-        return filter(cls, id=idlike).one()
-    elif isinstance(idlike, str):
-        qs = filter(cls, uid__startswith=idlike)
-        if issubclass(cls, IsVersioned):
-            if len(idlike) <= cls._len_stem_uid:
-                return qs.latest_version().one()
-            else:
-                return qs.one()
-        else:
-            return qs.one()
-    else:
-        assert idlike is None  # noqa: S101
-        # below behaves exactly like `.one()`
-        return cls.objects.get(**expressions)
+    return QuerySet(model=cls).get(idlike, **expressions)
 
 
 @classmethod  # type:ignore
@@ -393,9 +390,11 @@ def add_db_connection(db: str, using: str):
 @doc_args(Record.using.__doc__)
 def using(
     cls,
-    instance: str,
+    instance: str | None,
 ) -> QuerySet:
     """{}"""  # noqa: D415
+    if instance is None:
+        return QuerySet(model=cls, using=None)
     from lamindb_setup._connect_instance import (
         load_instance_settings,
         update_db_using_local,
@@ -541,24 +540,15 @@ def save(self, *args, **kwargs) -> Record:
         init_self_from_db(self, result)
     else:
         # save versioned record
-        if isinstance(self, IsVersioned) and self._is_new_version_of is not None:
-            if self._is_new_version_of.is_latest:
-                is_new_version_of = self._is_new_version_of
-            else:
-                # need one additional request
-                is_new_version_of = self.__class__.objects.get(
-                    is_latest=True, uid__startswith=self.stem_uid
-                )
-                logger.warning(
-                    f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
-                )
-            is_new_version_of.is_latest = False
+        if isinstance(self, IsVersioned) and self._revises is not None:
+            assert self._revises.is_latest  # noqa: S101
+            revises = self._revises
+            revises.is_latest = False
             with transaction.atomic():
-                is_new_version_of._is_new_version_of = (
-                    None  # ensure we don't start a recursion
-                )
-                is_new_version_of.save()
+                revises._revises = None  # ensure we don't start a recursion
+                revises.save()
                 super(Record, self).save(*args, **kwargs)
+            self._revises = None
         # save unversioned record
         else:
             super(Record, self).save(*args, **kwargs)

lamindb/_save.py

@@ -48,8 +48,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
 
     Examples:
 
-        Save a collection of records in one transaction, which is much faster
-        than writing a loop over ``projects.save()``:
+        Save a list of records:
 
         >>> labels = [ln.ULabel(f"Label {i}") for i in range(10)]
         >>> ln.save(projects)
@@ -61,7 +60,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
 
         Update a single existing record:
 
-        >>> transform = ln.filter(ln.Transform, uid="0Cb86EZj").one()
+        >>> transform = ln.Transform.get("0Cb86EZj")
         >>> transform.name = "New name"
         >>> transform.save()
 

lamindb/_transform.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Run, Transform
 
 from ._parents import _view_parents
 from ._run import delete_run_artifacts
-from .core.versioning import process_is_new_version_of
+from .core.versioning import message_update_key_in_version_family, process_revises
 
 if TYPE_CHECKING:
     from lnschema_core.types import TransformType
@@ -19,26 +20,38 @@ def __init__(transform: Transform, *args, **kwargs):
         return None
     name: str | None = kwargs.pop("name") if "name" in kwargs else None
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
-    is_new_version_of: Transform | None = (
-        kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
-    )
+    revises: Transform | None = kwargs.pop("revises") if "revises" in kwargs else None
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
     reference_type: str | None = (
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
     )
+    if "is_new_version_of" in kwargs:
+        logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
+        revises = kwargs.pop("is_new_version_of")
     # below is internal use that we'll hopefully be able to eliminate
     uid: str | None = kwargs.pop("uid") if "uid" in kwargs else None
     if not len(kwargs) == 0:
         raise ValueError(
-            "Only name, key, version, type, is_new_version_of, reference, "
+            "Only name, key, version, type, revises, reference, "
             f"reference_type can be passed, but you passed: {kwargs}"
         )
-    new_uid, version, name = process_is_new_version_of(
-        is_new_version_of, version, name, Transform
-    )
-    # this is only because the user-facing constructor allows passing an id
+    # Transform allows passing a uid, all others don't
+    if uid is None and key is not None:
+        revises = Transform.filter(key=key).order_by("-created_at").first()
+    if revises is not None and key is not None and revises.key != key:
+        note = message_update_key_in_version_family(
+            suid=revises.stem_uid,
+            existing_key=revises.key,
+            new_key=key,
+            registry="Artifact",
+        )
+        raise ValueError(
+            f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
+        )
+    new_uid, version, name, revises = process_revises(revises, version, name, Transform)
+    # this is only because the user-facing constructor allows passing a uid
     # most others don't
     if uid is None:
         has_consciously_provided_uid = False
@@ -54,7 +67,7 @@ def __init__(transform: Transform, *args, **kwargs):
         reference=reference,
         reference_type=reference_type,
         _has_consciously_provided_uid=has_consciously_provided_uid,
-        is_new_version_of=is_new_version_of,
+        revises=revises,
     )
 
 

lamindb/core/__init__.py

@@ -30,6 +30,7 @@ Curators:
 .. autosummary::
    :toctree: .
 
+   BaseCurator
    DataFrameCurator
    AnnDataCurator
    MuDataCurator
@@ -80,6 +81,7 @@ from lnschema_core.models import (
 
 from lamindb._curate import (
     AnnDataCurator,
+    BaseCurator,
     CurateLookup,
     DataFrameCurator,
     MuDataCurator,

lamindb/core/_context.py

@@ -18,7 +18,7 @@ from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
 from .exceptions import (
     MissingContext,
-    NotebookNotSaved,
+    NotebookFileNotSavedToDisk,
     NotebookNotSavedError,
     NoTitleError,
     TrackNotCalled,
@@ -26,7 +26,7 @@ from .exceptions import (
 )
 from .subsettings._transform_settings import transform_settings
 from .versioning import bump_version as bump_version_function
-from .versioning import increment_base62
+from .versioning import increment_base62, message_update_key_in_version_family
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
@@ -87,9 +87,9 @@ def raise_missing_context(transform_type: str, key: str) -> None:
         message = f"To track this {transform_type}, set\n\n"
     else:
         uid = transform.uid
-        suid, ruid = uid[: Transform._len_stem_uid], uid[Transform._len_stem_uid :]
-        new_ruid = increment_base62(ruid)
-        new_uid = f"{suid}{new_ruid}"
+        suid, vuid = uid[: Transform._len_stem_uid], uid[Transform._len_stem_uid :]
+        new_vuid = increment_base62(vuid)
+        new_uid = f"{suid}{new_vuid}"
         message = f"You already have a {transform_type} version family with key '{key}', suid '{transform.stem_uid}' & name '{transform.name}'.\n\n- to create a new {transform_type} version family, rename your file and rerun: ln.context.track()\n- to bump the version, set: "
     message += f'ln.context.uid = "{new_uid}"'
     if transform_type == "notebook":
@@ -199,7 +199,7 @@ class Context:
             :class:`~lamindb.Transform` object of ``type`` ``"pipeline"``:
 
             >>> ln.Transform(name="Cell Ranger", version="2", type="pipeline").save()
-            >>> transform = ln.Transform.filter(name="Cell Ranger", version="2").one()
+            >>> transform = ln.Transform.get(name="Cell Ranger", version="2")
             >>> ln.context.track(transform=transform)
         """
         self._path = None
@@ -224,7 +224,7 @@ class Context:
                             f"Please pass consistent version: ln.context.version = '{transform.version}'"
                         )
                     # test whether version was already used for another member of the family
-                    suid, ruid = (
+                    suid, vuid = (
                         self.uid[: Transform._len_stem_uid],
                         self.uid[Transform._len_stem_uid :],
                     )
@@ -233,7 +233,7 @@ class Context:
                     ).one_or_none()
                     if (
                         transform is not None
-                        and ruid != transform.uid[Transform._len_stem_uid :]
+                        and vuid != transform.uid[Transform._len_stem_uid :]
                     ):
                         better_version = bump_version_function(self.version)
                         raise SystemExit(
@@ -432,7 +432,12 @@ class Context:
                 suid = transform.stem_uid
                 new_suid = ids.base62_12()
                 transform_type = "Notebook" if is_run_from_ipython else "Script"
-                note = f'Or update key "{transform.key}" in your existing family:\n\nln.Transform.filter(uid__startswith="{suid}").update(key="{key}")'
+                note = message_update_key_in_version_family(
+                    suid=suid,
+                    existing_key=transform.key,
+                    new_key=key,
+                    registry="Transform",
+                )
                 raise UpdateContext(
                     f"{transform_type} filename changed.\n\nEither init a new transform family by setting:\n\n"
                     f'ln.context.uid = "{new_suid}0000"\n\n{note}'
@@ -460,14 +465,14 @@ class Context:
                         if is_run_from_ipython
                         else "Source code changed"
                     )
-                    suid, ruid = (
+                    suid, vuid = (
                         uid[: Transform._len_stem_uid],
                         uid[Transform._len_stem_uid :],
                     )
-                    new_ruid = increment_base62(ruid)
+                    new_vuid = increment_base62(vuid)
                     raise UpdateContext(
                         f"{change_type}, bump version by setting:\n\n"
-                        f'ln.context.uid = "{suid}{new_ruid}"'
+                        f'ln.context.uid = "{suid}{new_vuid}"'
                     )
             else:
                 self._logging_message += f"loaded Transform('{transform.uid}')"
@@ -499,8 +504,8 @@ class Context:
                 get_seconds_since_modified(context._path) > 3
                 and os.getenv("LAMIN_TESTING") is None
             ):
-                raise NotebookNotSaved(
-                    "Please save the notebook in your editor right before running `ln.finish()`"
+                raise NotebookFileNotSavedToDisk(
+                    "Please save the notebook manually in your editor right before running `ln.finish()`"
                 )
         save_context_core(
             run=context.run,

lamindb/core/_feature_manager.py

@@ -361,9 +361,7 @@ def __getitem__(self, slot) -> QuerySet:
     return getattr(feature_set, self._accessor_by_registry[orm_name]).all()
 
 
-@classmethod  # type: ignore
-def filter(cls, **expression) -> QuerySet:
-    """Filter features."""
+def filter_base(cls, **expression):
     if cls in {FeatureManagerArtifact, FeatureManagerCollection}:
         model = Feature
         value_model = FeatureValue
@@ -379,14 +377,20 @@ def filter(cls, **expression) -> QuerySet:
     new_expression = {}
     features = model.filter(name__in=keys_normalized).all().distinct()
     for key, value in expression.items():
-        normalized_key = key.split("__")[0]
+        split_key = key.split("__")
+        normalized_key = split_key[0]
+        comparator = ""
+        if len(split_key) == 2:
+            comparator = f"__{split_key[1]}"
         feature = features.get(name=normalized_key)
         if not feature.dtype.startswith("cat"):
-            feature_value = value_model.filter(feature=feature, value=value).one()
-            new_expression["_feature_values"] = feature_value
+            expression = {"feature": feature, f"value{comparator}": value}
+            feature_value = value_model.filter(**expression)
+            new_expression["_feature_values__in"] = feature_value
         else:
             if isinstance(value, str):
-                label = ULabel.filter(name=value).one()
+                expression = {f"name{comparator}": value}
+                label = ULabel.get(**expression)
                 new_expression["ulabels"] = label
             else:
                 raise NotImplementedError
@@ -398,6 +402,18 @@ def filter(cls, **expression) -> QuerySet:
         return Run.filter(**new_expression)
 
 
+@classmethod  # type: ignore
+def filter(cls, **expression) -> QuerySet:
+    """Query artifacts by features."""
+    return filter_base(cls, **expression)
+
+
+@classmethod  # type: ignore
+def get(cls, **expression) -> Record:
+    """Query a single artifact by feature."""
+    return filter_base(cls, **expression).one()
+
+
 @property  # type: ignore
 def _feature_set_by_slot(self):
     """Feature sets by slot."""
@@ -474,7 +490,7 @@ def _add_values(
     _feature_values = []
     not_validated_values = []
     for key, value in features_values.items():
-        feature = model.filter(name=key).one()
+        feature = model.get(name=key)
         inferred_type, converted_value = infer_feature_type_convert_json(
             value,
             mute=True,
@@ -843,5 +859,6 @@ FeatureManager._add_set_from_anndata = _add_set_from_anndata
 FeatureManager._add_set_from_mudata = _add_set_from_mudata
 FeatureManager._add_from = _add_from
 FeatureManager.filter = filter
+FeatureManager.get = get
 ParamManager.add_values = add_values_params
 ParamManager.get_values = get_values

lamindb/core/_label_manager.py

@@ -234,7 +234,7 @@ class LabelManager:
                 if hasattr(self._host, related_name):
                     for feature_name, labels in labels_by_features.items():
                         if feature_name is not None:
-                            feature_id = Feature.filter(name=feature_name).one().id
+                            feature_id = Feature.get(name=feature_name).id
                         else:
                             feature_id = None
                         getattr(self._host, related_name).add(

lamindb/core/_mapped_collection.py

@@ -17,6 +17,7 @@ from .storage._anndata_accessor import (
     GroupTypes,
     StorageType,
     _safer_read_index,
+    get_spec,
     registry,
 )
 
@@ -153,13 +154,30 @@ class MappedCollection:
         self._make_connections(path_list, parallel)
 
         self.n_obs_list = []
-        for storage in self.storages:
+        for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
                 X = store["X"]
+                store_path = self.path_list[i]
+                self._check_csc_raise_error(X, "X", store_path)
                 if isinstance(X, ArrayTypes):  # type: ignore
                     self.n_obs_list.append(X.shape[0])
                 else:
                     self.n_obs_list.append(X.attrs["shape"][0])
+                for layer_key in self.layers_keys:
+                    if layer_key == "X":
+                        continue
+                    self._check_csc_raise_error(
+                        store["layers"][layer_key],
+                        f"layers/{layer_key}",
+                        store_path,
+                    )
+                if self.obsm_keys is not None:
+                    for obsm_key in self.obsm_keys:
+                        self._check_csc_raise_error(
+                            store["obsm"][obsm_key],
+                            f"obsm/{obsm_key}",
+                            store_path,
+                        )
         self.n_obs = sum(self.n_obs_list)
 
         self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
@@ -281,6 +299,18 @@ class MappedCollection:
         vars = pd.Index(vars)
         return [i for i, vrs in enumerate(self.var_list) if not vrs.equals(vars)]
 
+    def _check_csc_raise_error(
+        self, elem: GroupType | ArrayType, key: str, path: UPathStr
+    ):
+        if isinstance(elem, ArrayTypes):  # type: ignore
+            return
+        if get_spec(elem).encoding_type == "csc_matrix":
+            if not self.parallel:
+                self.close()
+            raise ValueError(
+                f"{key} in {path} is a csc matrix, `MappedCollection` doesn't support this format yet."
+            )
+
     def __len__(self):
         return self.n_obs
 

lamindb/core/exceptions.py

@@ -20,7 +20,7 @@ class TrackNotCalled(SystemExit):
     pass
 
 
-class NotebookNotSaved(SystemExit):
+class NotebookFileNotSavedToDisk(SystemExit):
     pass
 
 

lamindb/core/storage/__init__.py

@@ -18,8 +18,8 @@ Array accessors.
 
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
-from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor
+from ._tiledbsoma import save_tiledbsoma_experiment
 from ._valid_suffixes import VALID_SUFFIXES
 from .objects import infer_suffix, write_to_disk
 from .paths import delete_storage, load_to_memory

lamindb/core/storage/_backed_access.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Callable, Literal
+from typing import TYPE_CHECKING, Any, Callable
 
 from anndata._io.specs.registry import get_spec
 from lnschema_core import Artifact
 
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
+from ._tiledbsoma import _open_tiledbsoma
 from .paths import filepath_from_artifact
 
 if TYPE_CHECKING:
@@ -52,43 +53,6 @@ def _track_writes_factory(obj: Any, finalize: Callable):
     return obj
 
 
-def _open_tiledbsoma(
-    filepath: UPath, mode: Literal["r", "w"] = "r"
-) -> SOMACollection | SOMAExperiment:
-    try:
-        import tiledbsoma as soma
-    except ImportError as e:
-        raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
-    filepath_str = filepath.as_posix()
-    if filepath.protocol == "s3":
-        from lamindb_setup.core._settings_storage import get_storage_region
-
-        region = get_storage_region(filepath_str)
-        tiledb_config = {"vfs.s3.region": region}
-        storage_options = filepath.storage_options
-        if "key" in storage_options:
-            tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
-        if "secret" in storage_options:
-            tiledb_config["vfs.s3.aws_secret_access_key"] = storage_options["secret"]
-        if "token" in storage_options:
-            tiledb_config["vfs.s3.aws_session_token"] = storage_options["token"]
-        ctx = soma.SOMATileDBContext(tiledb_config=tiledb_config)
-        # this is a strange bug
-        # for some reason iterdir futher gives incorrect results
-        # if cache is not invalidated
-        # instead of obs and ms it gives ms and ms in the list of names
-        filepath.fs.invalidate_cache()
-    else:
-        ctx = None
-
-    soma_objects = [obj.name for obj in filepath.iterdir()]
-    if "obs" in soma_objects and "ms" in soma_objects:
-        SOMAType = soma.Experiment
-    else:
-        SOMAType = soma.Collection
-    return SOMAType.open(filepath_str, mode=mode, context=ctx)
-
-
 @dataclass
 class BackedAccessor:
     """h5py.File or zarr.Group accessor."""