lamindb 0.74.3__py3-none-any.whl → 0.75.1__py3-none-any.whl

lamindb/__init__.py

@@ -42,7 +42,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.74.3"
+__version__ = "0.75.1"
 
 import os as _os
 

lamindb/_artifact.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import shutil
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, Mapping
@@ -28,6 +29,7 @@ from lnschema_core.types import (
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import HasFeatures, _track_run_input
 from lamindb.core._settings import settings
+from lamindb.core.exceptions import IntegrityError
 from lamindb.core.storage import (
     LocalPathClasses,
     UPath,
@@ -39,6 +41,7 @@ from lamindb.core.storage import (
 from lamindb.core.storage.paths import (
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
+    check_path_is_child_of_root,
     filepath_from_artifact,
 )
 from lamindb.core.versioning import get_uid_from_old_version, init_uid
@@ -102,7 +105,11 @@ def process_pathlike(
             if not isinstance(filepath, LocalPathClasses):
                 # for a cloud path, new_root is always the bucket name
                 new_root = list(filepath.parents)[-1]
-                storage_settings = init_storage(new_root)
+                # do not register remote storage locations on hub if the current instance
+                # is not managed on the hub
+                storage_settings = init_storage(
+                    new_root, prevent_register_hub=not setup_settings.instance.is_on_hub
+                )
                 storage_record = register_storage_in_instance(storage_settings)
                 use_existing_storage_key = True
                 return storage_record, use_existing_storage_key
@@ -257,14 +264,6 @@ def check_path_in_existing_storage(
     return False
 
 
-def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
-    # str is needed to eliminate UPath storage_options
-    # from the equality checks below
-    path = UPath(str(path))
-    root = UPath(str(root))
-    return root.resolve() in path.resolve().parents
-
-
 def get_relative_path_to_directory(
     path: PurePath | Path | UPath, directory: PurePath | Path | UPath
 ) -> PurePath | Path:
@@ -343,8 +342,10 @@ def get_artifact_kwargs_from_data(
     else:
         storage = default_storage
 
-    if key is not None and key.startswith(AUTO_KEY_PREFIX):
-        raise ValueError(f"Key cannot start with {AUTO_KEY_PREFIX}")
+    # for now comment out this error to allow creating new versions of stores
+    # in the default folder (.lamindb)
+    #    if key is not None and key.startswith(AUTO_KEY_PREFIX):
+    #        raise ValueError(f"Key cannot start with {AUTO_KEY_PREFIX}")
 
     log_storage_hint(
         check_path_in_storage=check_path_in_storage,
@@ -366,7 +367,7 @@ def get_artifact_kwargs_from_data(
     kwargs = {
         "suffix": suffix,
         "hash": hash,
-        "hash_type": hash_type,
+        "_hash_type": hash_type,
         "key": key,
         "size": size,
         "storage_id": storage.id,
@@ -377,7 +378,7 @@ def get_artifact_kwargs_from_data(
         "n_observations": None,  # to implement
         "run_id": run.id if run is not None else None,
         "run": run,
-        "key_is_virtual": key_is_virtual,
+        "_key_is_virtual": key_is_virtual,
     }
     if not isinstance(path, LocalPathClasses):
         local_filepath = None
@@ -502,7 +503,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed: data")
 
     data: str | Path = kwargs.pop("data") if len(args) == 0 else args[0]
-    type: str = kwargs.pop("type") if "type" in kwargs else "dataset"
+    type: str = kwargs.pop("type") if "type" in kwargs else None
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     description: str | None = (
@@ -531,7 +532,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
     using_key = (
         kwargs.pop("using_key") if "using_key" in kwargs else settings._using_key
     )
-    accessor = kwargs.pop("accessor") if "accessor" in kwargs else None
+    accessor = kwargs.pop("_accessor") if "_accessor" in kwargs else None
     accessor = _check_accessor_artifact(data=data, accessor=accessor)
     if not len(kwargs) == 0:
         raise ValueError(
@@ -592,7 +593,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
     kwargs["version"] = version
     kwargs["description"] = description
     kwargs["visibility"] = visibility
-    kwargs["accessor"] = accessor
+    kwargs["_accessor"] = accessor
     # this check needs to come down here because key might be populated from an
     # existing file path during get_artifact_kwargs_from_data()
     if (
@@ -633,7 +634,7 @@ def from_df(
         description=description,
         version=version,
         is_new_version_of=is_new_version_of,
-        accessor="DataFrame",
+        _accessor="DataFrame",
         type="dataset",
         **kwargs,
     )
@@ -662,7 +663,7 @@ def from_anndata(
         description=description,
         version=version,
         is_new_version_of=is_new_version_of,
-        accessor="AnnData",
+        _accessor="AnnData",
         type="dataset",
         **kwargs,
     )
@@ -689,7 +690,7 @@ def from_mudata(
         description=description,
         version=version,
         is_new_version_of=is_new_version_of,
-        accessor="MuData",
+        _accessor="MuData",
         type="dataset",
         **kwargs,
     )
@@ -707,8 +708,8 @@ def from_dir(
 ) -> list[Artifact]:
     """{}"""  # noqa: D415
     logger.warning(
-        "this creates one artifact per file in the directory - you might simply call"
-        " ln.Artifact(dir) to get one artifact for the entire directory"
+        "this creates one artifact per file in the directory - consider"
+        " ln.Artifact(dir_path) to get one artifact for the entire directory"
     )
     folderpath: UPath = create_path(path)  # returns Path for local
     default_storage = settings._storage_settings.record
@@ -823,7 +824,7 @@ def replace(
     if check_path_in_storage:
         raise ValueError("Can only replace with a local file not in any Storage.")
 
-    if self.key is not None and not self.key_is_virtual:
+    if self.key is not None and not self._key_is_virtual:
         key_path = PurePosixPath(self.key)
         new_filename = f"{key_path.stem}{kwargs['suffix']}"
         # the following will only be true if the suffix changes!
@@ -849,7 +850,7 @@ def replace(
     self.suffix = kwargs["suffix"]
     self.size = kwargs["size"]
     self.hash = kwargs["hash"]
-    self.hash_type = kwargs["hash_type"]
+    self._hash_type = kwargs["_hash_type"]
     self.run_id = kwargs["run_id"]
     self.run = kwargs["run"]
 
@@ -862,15 +863,15 @@ def replace(
 
 # deprecated
 def backed(
-    self, is_run_input: bool | None = None
+    self, mode: str = "r", is_run_input: bool | None = None
 ) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
     logger.warning("`.backed()` is deprecated, use `.open()`!'")
-    return self.open(is_run_input)
+    return self.open(mode, is_run_input)
 
 
 # docstring handled through attach_func_to_class_method
 def open(
-    self, is_run_input: bool | None = None
+    self, mode: str = "r", is_run_input: bool | None = None
 ) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
     # ignore empty suffix for now
     suffixes = (".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma", "")
@@ -880,29 +881,61 @@ def open(
             " use one of the following suffixes for the object name:"
             f" {', '.join(suffixes[:-1])}."
         )
+    if self.suffix != ".tiledbsoma" and self.key != "soma" and mode != "r":
+        raise ValueError("Only a tiledbsoma store can be openened with `mode!='r'`.")
 
-    from lamindb.core.storage._backed_access import backed_access
+    from lamindb.core.storage._backed_access import _track_writes_factory, backed_access
 
-    _track_run_input(self, is_run_input)
     using_key = settings._using_key
     filepath = filepath_from_artifact(self, using_key=using_key)
+    is_tiledbsoma_w = (
+        filepath.name == "soma" or filepath.suffix == ".tiledbsoma"
+    ) and mode == "w"
     # consider the case where an object is already locally cached
     localpath = setup_settings.instance.storage.cloud_to_local_no_update(filepath)
-    if localpath.exists():
-        return backed_access(localpath, using_key)
+    if not is_tiledbsoma_w and localpath.exists():
+        access = backed_access(localpath, mode, using_key)
     else:
-        return backed_access(filepath, using_key)
+        access = backed_access(filepath, mode, using_key)
+        if is_tiledbsoma_w:
+
+            def finalize():
+                nonlocal self, filepath, localpath
+                if not isinstance(filepath, LocalPathClasses):
+                    _, hash, _, _ = get_stat_dir_cloud(filepath)
+                else:
+                    # this can be very slow
+                    _, hash, _, _ = hash_dir(filepath)
+                if self.hash != hash:
+                    from ._record import init_self_from_db
+
+                    logger.warning(
+                        "The hash of the tiledbsoma store has changed, creating a new version of the artifact."
+                    )
+                    new_version = Artifact(filepath, is_new_version_of=self).save()
+                    init_self_from_db(self, new_version)
+
+                    if localpath != filepath and localpath.exists():
+                        shutil.rmtree(localpath)
+
+            access = _track_writes_factory(access, finalize)
+    # only call if open is successfull
+    _track_run_input(self, is_run_input)
+    return access
 
 
 # docstring handled through attach_func_to_class_method
 def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs) -> Any:
-    _track_run_input(self, is_run_input)
     if hasattr(self, "_memory_rep") and self._memory_rep is not None:
-        return self._memory_rep
-    using_key = settings._using_key
-    return load_to_memory(
-        filepath_from_artifact(self, using_key=using_key), stream=stream, **kwargs
-    )
+        access_memory = self._memory_rep
+    else:
+        using_key = settings._using_key
+        access_memory = load_to_memory(
+            filepath_from_artifact(self, using_key=using_key), stream=stream, **kwargs
+        )
+    # only call if load is successfull
+    _track_run_input(self, is_run_input)
+    return access_memory
 
 
 # docstring handled through attach_func_to_class_method
@@ -935,6 +968,17 @@ def delete(
     storage: bool | None = None,
     using_key: str | None = None,
 ) -> None:
+    # this first check means an invalid delete fails fast rather than cascading through
+    # database and storage permission errors
+    if os.getenv("LAMINDB_MULTI_INSTANCE") is None:
+        isettings = setup_settings.instance
+        if self.storage.instance_uid != isettings.uid and (storage or storage is None):
+            raise IntegrityError(
+                "Cannot simply delete artifacts outside of this instance's managed storage locations."
+                "\n(1) If you only want to delete the metadata record in this instance, pass `storage=False`"
+                f"\n(2) If you want to delete the artifact in storage, please load the managing lamindb instance (uid={self.storage.instance_uid})."
+                f"\nThese are all managed storage locations of this instance:\n{Storage.filter(instance_uid=isettings.uid).df()}"
+            )
     # by default, we only move artifacts into the trash (visibility = -1)
     trash_visibility = VisibilityChoice.trash.value
     if self.visibility > trash_visibility and not permanent:
@@ -943,7 +987,7 @@ def delete(
         # move to trash
         self.visibility = trash_visibility
         self.save()
-        logger.warning(f"moved artifact to trash (visibility = {trash_visibility})")
+        logger.important(f"moved artifact to trash (visibility = {trash_visibility})")
         return
 
     # if the artifact is already in the trash
@@ -970,7 +1014,7 @@ def delete(
         # only delete in storage if DB delete is successful
         # DB delete might error because of a foreign key constraint violated etc.
         self._delete_skip_storage()
-        if self.key is None or self.key_is_virtual:
+        if self.key is None or self._key_is_virtual:
             # do not ask for confirmation also if storage is None
             delete_in_storage = storage is None or storage
         else:
@@ -985,9 +1029,7 @@ def delete(
             else:
                 delete_in_storage = storage
         if not delete_in_storage:
-            logger.warning(
-                f"you will retain a dangling store here: {path}, not referenced via an artifact"
-            )
+            logger.important(f"a file/folder remains here: {path}")
         # we don't yet have logic to bring back the deleted metadata record
         # in case storage deletion fails - this is important for ACID down the road
         if delete_in_storage:
@@ -1013,7 +1055,7 @@ def save(self, upload: bool | None = None, **kwargs) -> Artifact:
         self._local_filepath = local_path
         # switch to virtual storage key upon upload
         # the local filepath is already cached at that point
-        self.key_is_virtual = True
+        self._key_is_virtual = True
         # ensure that the artifact is uploaded
         self._to_store = True
 

lamindb/_can_validate.py

@@ -13,7 +13,7 @@ from lnschema_core import CanValidate, Record
 from lamindb._utils import attach_func_to_class_method
 
 from ._from_values import _has_organism_field, _print_values
-from ._record import _queryset, get_default_str_field
+from ._record import _queryset, get_name_field
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -30,7 +30,7 @@ def inspect(
     *,
     mute: bool = False,
     organism: str | Record | None = None,
-    public_source: Record | None = None,
+    source: Record | None = None,
 ) -> InspectResult:
     """{}"""  # noqa: D415
     return _inspect(
@@ -39,7 +39,7 @@ def inspect(
         field=field,
         mute=mute,
         organism=organism,
-        public_source=public_source,
+        source=source,
     )
 
 
@@ -52,9 +52,31 @@ def validate(
     *,
     mute: bool = False,
     organism: str | Record | None = None,
+    source: Record | None = None,
 ) -> np.ndarray:
     """{}"""  # noqa: D415
-    return _validate(cls=cls, values=values, field=field, mute=mute, organism=organism)
+    return _validate(
+        cls=cls, values=values, field=field, mute=mute, organism=organism, source=source
+    )
+
+
+def _check_source_db(source: Record, using_key: str | None):
+    """Check if the source is from the DB."""
+    if using_key is not None and using_key != "default":
+        if source._state.db != using_key:
+            raise ValueError(
+                f"source must be a bionty.Source record from instance '{using_key}'!"
+            )
+
+
+def _check_organism_db(organism: Record, using_key: str | None):
+    """Check if the organism is from the DB."""
+    if isinstance(organism, Record):
+        if using_key is not None and using_key != "default":
+            if organism._state.db != using_key:
+                raise ValueError(
+                    f"organism must be a bionty.Organism record from instance '{using_key}'!"
+                )
 
 
 def _inspect(
@@ -65,7 +87,7 @@ def _inspect(
     mute: bool = False,
     using_key: str | None = None,
     organism: str | Record | None = None,
-    public_source: Record | None = None,
+    source: Record | None = None,
 ) -> pd.DataFrame | dict[str, list[str]]:
     """{}"""  # noqa: D415
     from lamin_utils._inspect import inspect
@@ -73,10 +95,15 @@ def _inspect(
     if isinstance(values, str):
         values = [values]
 
-    field = get_default_str_field(cls, field=field)
+    field = get_name_field(cls, field=field)
     queryset = _queryset(cls, using_key)
-    orm = queryset.model
-    model_name = orm._meta.model.__name__
+    using_key = queryset.db
+    if isinstance(source, Record):
+        _check_source_db(source, using_key)
+        queryset = queryset.filter(source=source).all()
+    _check_organism_db(organism, using_key)
+    registry = queryset.model
+    model_name = registry._meta.model.__name__
 
     # inspect in the DB
     result_db = inspect(
@@ -89,11 +116,11 @@ def _inspect(
     )
     nonval = set(result_db.non_validated).difference(result_db.synonyms_mapper.keys())
 
-    if len(nonval) > 0 and orm.__get_schema_name__() == "bionty":
+    if len(nonval) > 0 and registry.__get_schema_name__() == "bionty":
         try:
-            bionty_result = orm.public(
-                organism=organism, public_source=public_source
-            ).inspect(values=nonval, field=field, mute=True)
+            bionty_result = registry.public(organism=organism, source=source).inspect(
+                values=nonval, field=field, mute=True
+            )
             bionty_validated = bionty_result.validated
             bionty_mapper = bionty_result.synonyms_mapper
             hint = False
@@ -135,7 +162,7 @@ def _inspect(
         logger.print(f"   couldn't validate {labels}: {colors.red(print_values)}")
         logger.print(
             f"→  if you are sure, create new record{s} via"
-            f" {colors.italic(f'ln.{orm.__name__}()')} and save to your registry"
+            f" {colors.italic(f'{registry.__name__}()')} and save to your registry"
         )
 
     return result_db
@@ -149,6 +176,7 @@ def _validate(
     mute: bool = False,
     using_key: str | None = None,
     organism: str | Record | None = None,
+    source: Record | None = None,
 ) -> np.ndarray:
     """{}"""  # noqa: D415
     from lamin_utils._inspect import validate
@@ -157,9 +185,14 @@ def _validate(
     if isinstance(values, str):
         values = [values]
 
-    field = get_default_str_field(cls, field=field)
+    field = get_name_field(cls, field=field)
 
     queryset = _queryset(cls, using_key)
+    using_key = queryset.db
+    if isinstance(source, Record):
+        _check_source_db(source, using_key)
+        queryset = queryset.filter(source=source).all()
+    _check_organism_db(organism, using_key)
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
@@ -169,6 +202,15 @@ def _validate(
         ),
         dtype="object",
     )
+    if field_values.empty:
+        if not mute:
+            msg = (
+                f"Your {cls.__name__} registry is empty, consider populating it first!"
+            )
+            if hasattr(cls, "source_id"):
+                msg += "\n   → use `.import_from_source()` to import records from a source, e.g. a public ontology"
+            logger.warning(msg)
+        return np.array([False] * len(values))
 
     result = validate(
         identifiers=values,
@@ -198,6 +240,7 @@ def standardize(
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
     organism: str | Record | None = None,
+    source: Record | None = None,
 ) -> list[str] | dict[str, str]:
     """{}"""  # noqa: D415
     return _standardize(
@@ -212,6 +255,7 @@ def standardize(
         keep=keep,
         synonyms_field=synonyms_field,
         organism=organism,
+        source=source,
     )
 
 
@@ -263,6 +307,7 @@ def _standardize(
     synonyms_field: str = "synonyms",
     using_key: str | None = None,
     organism: str | Record | None = None,
+    source: Record | None = None,
 ) -> list[str] | dict[str, str]:
     """{}"""  # noqa: D415
     from lamin_utils._standardize import standardize as map_synonyms
@@ -271,24 +316,29 @@ def _standardize(
     if isinstance(values, str):
         values = [values]
 
-    field = get_default_str_field(cls, field=field)
-    return_field = get_default_str_field(
+    field = get_name_field(cls, field=field)
+    return_field = get_name_field(
         cls, field=field if return_field is None else return_field
     )
     queryset = _queryset(cls, using_key)
-    orm = queryset.model
-
-    if _has_organism_field(orm):
-        # here, we can safely import lnschema_bionty
-        from lnschema_bionty._bionty import create_or_get_organism_record
-
-        organism_record = create_or_get_organism_record(organism=organism, orm=orm)
+    using_key = queryset.db
+    if isinstance(source, Record):
+        _check_source_db(source, using_key)
+        queryset = queryset.filter(source=source).all()
+    _check_organism_db(organism, using_key)
+    registry = queryset.model
+
+    if _has_organism_field(registry):
+        # here, we can safely import bionty
+        from bionty._bionty import create_or_get_organism_record
+
+        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
         organism = (
             organism_record.name if organism_record is not None else organism_record
         )
 
     try:
-        orm._meta.get_field(synonyms_field)
+        registry._meta.get_field(synonyms_field)
         df = _filter_query_based_on_organism(
             queryset=queryset, field=field, organism=organism
         )
@@ -320,7 +370,7 @@ def _standardize(
             return result
 
     # map synonyms in Bionty
-    if orm.__get_schema_name__() == "bionty" and public_aware:
+    if registry.__get_schema_name__() == "bionty" and public_aware:
         mapper = {}
         if return_mapper:
             mapper = std_names_db
@@ -328,12 +378,14 @@ def _standardize(
                 df=df, identifiers=values, return_mapper=False, mute=True, **_kwargs
             )
 
-        val_res = orm.validate(std_names_db, field=field, mute=True, organism=organism)
+        val_res = registry.validate(
+            std_names_db, field=field, mute=True, organism=organism
+        )
         if all(val_res):
             return _return(result=std_names_db, mapper=mapper)
 
         nonval = np.array(std_names_db)[~val_res]
-        std_names_bt_mapper = orm.public(organism=organism).standardize(
+        std_names_bt_mapper = registry.public(organism=organism).standardize(
             nonval, return_mapper=True, mute=True, **_kwargs
         )
 
@@ -345,7 +397,7 @@ def _standardize(
                 f" {list(std_names_bt_mapper.keys())}"
             )
             warn_msg += (
-                f"\n   please add corresponding {orm._meta.model.__name__} records via"
+                f"\n   please add corresponding {registry._meta.model.__name__} records via"
                 f" `.from_values({list(set(std_names_bt_mapper.values()))})`"
             )
             logger.warning(warn_msg)
@@ -388,7 +440,10 @@ def _add_or_remove_synonyms(
                 " with the following records:\n"
             )
             display(records_df)
-            raise SystemExit(AssertionError)
+            raise ValueError(
+                "cannot assigned a synonym that is already associated with a record to a different record.\n"
+                "Consider removing the synonym from existing records or using a different synonym."
+            )
 
     # passed synonyms
     # nothing happens when passing an empty string or list
@@ -405,7 +460,7 @@ def _add_or_remove_synonyms(
         return
     # because we use | as the separator
     if any("|" in i for i in syn_new_set):
-        raise AssertionError("a synonym can't contain '|'!")
+        raise ValueError("a synonym can't contain '|'!")
 
     # existing synonyms
     syns_exist = record.synonyms
@@ -453,13 +508,13 @@ def _filter_query_based_on_organism(
     """Filter a queryset based on organism."""
     import pandas as pd
 
-    orm = queryset.model
+    registry = queryset.model
 
-    if _has_organism_field(orm) and not field.endswith("id"):
-        # here, we can safely import lnschema_bionty
-        from lnschema_bionty._bionty import create_or_get_organism_record
+    if _has_organism_field(registry) and not _field_is_id(field, registry):
+        # here, we can safely import bionty
+        from bionty._bionty import create_or_get_organism_record
 
-        organism_record = create_or_get_organism_record(organism=organism, orm=orm)
+        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
         if organism_record is not None:
             queryset = queryset.filter(organism__name=organism_record.name)
 
@@ -469,6 +524,16 @@ def _filter_query_based_on_organism(
         return queryset.values_list(values_list_field, flat=True)
 
 
+def _field_is_id(field: str, registry: type[Record]) -> bool:
+    """Check if the field is an ontology ID."""
+    if hasattr(registry, "_ontology_id_field"):
+        if field == registry._ontology_id_field:
+            return True
+    if field.endswith("id"):
+        return True
+    return False
+
+
 METHOD_NAMES = [
     "validate",
     "inspect",