lamindb 0.76.7__py3-none-any.whl → 0.76.9__py3-none-any.whl

lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.7"
+__version__ = "0.76.9"
 
 import os as _os
 

lamindb/_artifact.py

@@ -30,7 +30,7 @@ from lnschema_core.types import (
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input, describe, view_lineage
 from lamindb.core._settings import settings
-from lamindb.core.exceptions import IntegrityError
+from lamindb.core.exceptions import IntegrityError, InvalidArgument
 from lamindb.core.loaders import load_to_memory
 from lamindb.core.storage import (
     LocalPathClasses,
@@ -43,6 +43,7 @@ from lamindb.core.storage.paths import (
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     check_path_is_child_of_root,
+    filepath_cache_key_from_artifact,
     filepath_from_artifact,
 )
 from lamindb.core.versioning import (
@@ -88,8 +89,6 @@ def process_pathlike(
                 raise FileNotFoundError(filepath)
         except PermissionError:
             pass
-    if isinstance(filepath, LocalPathClasses):
-        filepath = filepath.resolve()
     if check_path_is_child_of_root(filepath, default_storage.root):
         use_existing_storage_key = True
         return default_storage, use_existing_storage_key
@@ -154,7 +153,7 @@ def process_data(
             if hasattr(default_storage, "_access_token")
             else None
         )
-        path = create_path(data, access_token=access_token)
+        path = create_path(data, access_token=access_token).resolve()
         storage, use_existing_storage_key = process_pathlike(
             path,
             default_storage=default_storage,
@@ -175,12 +174,12 @@ def process_data(
             key_suffix = None
         suffix = infer_suffix(data, format)
         if key_suffix is not None and key_suffix != suffix:
-            raise ValueError(
+            raise InvalidArgument(
                 f"The suffix '{key_suffix}' of the provided key is incorrect, it should"
                 f" be '{suffix}'."
             )
         cache_name = f"{provisional_uid}{suffix}"
-        path = settings._storage_settings.cache_dir / cache_name
+        path = settings.storage.cache_dir / cache_name
         # Alex: I don't understand the line below
         if path.suffixes == []:
             path = path.with_suffix(suffix)
@@ -344,8 +343,8 @@ def get_artifact_kwargs_from_data(
 
     if revises is not None:  # update provisional_uid
         provisional_uid, revises = create_uid(revises=revises, version=version)
-        if path.as_posix().startswith(settings._storage_settings.cache_dir.as_posix()):
-            path = path.rename(f"{provisional_uid}{suffix}")
+        if settings.storage.cache_dir in path.parents:
+            path = path.rename(path.with_name(f"{provisional_uid}{suffix}"))
 
     check_path_in_storage = False
     if use_existing_storage_key:
@@ -356,7 +355,7 @@ def get_artifact_kwargs_from_data(
             key = inferred_key
         else:
             if not key == inferred_key:
-                raise ValueError(
+                raise InvalidArgument(
                     f"The path '{data}' is already in registered storage"
                     f" '{storage.root}' with key '{inferred_key}'\nYou passed"
                     f" conflicting key '{key}': please move the file before"
@@ -399,6 +398,7 @@ def get_artifact_kwargs_from_data(
         "run_id": run.id if run is not None else None,
         "run": run,
         "_key_is_virtual": key_is_virtual,
+        "revises": revises,
     }
     if not isinstance(path, LocalPathClasses):
         local_filepath = None
@@ -614,6 +614,9 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    if revises is None:
+        revises = kwargs_or_artifact.pop("revises")
+
     if data is not None:
         artifact._local_filepath = privates["local_filepath"]
         artifact._cloud_filepath = privates["cloud_filepath"]
@@ -750,7 +753,7 @@ def from_dir(
         " ln.Artifact(dir_path) to get one artifact for the entire directory"
     )
     folderpath: UPath = create_path(path)  # returns Path for local
-    default_storage = settings._storage_settings.record
+    default_storage = settings.storage.record
     using_key = settings._using_key
     storage, use_existing_storage = process_pathlike(
         folderpath, default_storage, using_key
@@ -844,7 +847,7 @@ def replace(
     run: Run | None = None,
     format: str | None = None,
 ) -> None:
-    default_storage = settings._storage_settings.record
+    default_storage = settings.storage.record
     kwargs, privates = get_artifact_kwargs_from_data(
         provisional_uid=self.uid,
         data=data,
@@ -919,12 +922,14 @@ def open(
     from lamindb.core.storage._backed_access import _track_writes_factory, backed_access
 
     using_key = settings._using_key
-    filepath = filepath_from_artifact(self, using_key=using_key)
+    filepath, cache_key = filepath_cache_key_from_artifact(self, using_key=using_key)
     is_tiledbsoma_w = (
         filepath.name == "soma" or filepath.suffix == ".tiledbsoma"
     ) and mode == "w"
     # consider the case where an object is already locally cached
-    localpath = setup_settings.instance.storage.cloud_to_local_no_update(filepath)
+    localpath = setup_settings.instance.storage.cloud_to_local_no_update(
+        filepath, cache_key=cache_key
+    )
     if not is_tiledbsoma_w and localpath.exists():
         access = backed_access(localpath, mode, using_key)
     else:
@@ -956,15 +961,17 @@ def open(
 
 
 # can't really just call .cache in .load because of double tracking
-def _synchronize_cleanup_on_error(filepath: UPath) -> UPath:
+def _synchronize_cleanup_on_error(
+    filepath: UPath, cache_key: str | None = None
+) -> UPath:
     try:
         cache_path = setup_settings.instance.storage.cloud_to_local(
-            filepath, print_progress=True
+            filepath, cache_key=cache_key, print_progress=True
         )
     except Exception as e:
         if not isinstance(filepath, LocalPathClasses):
             cache_path = setup_settings.instance.storage.cloud_to_local_no_update(
-                filepath
+                filepath, cache_key=cache_key
             )
             if cache_path.is_file():
                 cache_path.unlink(missing_ok=True)
@@ -979,8 +986,11 @@ def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
     if hasattr(self, "_memory_rep") and self._memory_rep is not None:
         access_memory = self._memory_rep
     else:
-        filepath = filepath_from_artifact(self, using_key=settings._using_key)
-        cache_path = _synchronize_cleanup_on_error(filepath)
+        filepath, cache_key = filepath_cache_key_from_artifact(
+            self, using_key=settings._using_key
+        )
+        cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
+        # cache_path is local so doesn't trigger any sync in load_to_memory
         access_memory = load_to_memory(cache_path, **kwargs)
     # only call if load is successfull
     _track_run_input(self, is_run_input)
@@ -989,8 +999,10 @@ def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
 
 # docstring handled through attach_func_to_class_method
 def cache(self, is_run_input: bool | None = None) -> Path:
-    filepath = filepath_from_artifact(self, using_key=settings._using_key)
-    cache_path = _synchronize_cleanup_on_error(filepath)
+    filepath, cache_key = filepath_cache_key_from_artifact(
+        self, using_key=settings._using_key
+    )
+    cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
     # only call if sync is successfull
     _track_run_input(self, is_run_input)
     return cache_path
@@ -1041,7 +1053,7 @@ def delete(
     if delete_record:
         # need to grab file path before deletion
         try:
-            path = filepath_from_artifact(self, using_key)
+            path, _ = filepath_from_artifact(self, using_key)
         except OSError:
             # we can still delete the record
             logger.warning("Could not get path")
@@ -1133,8 +1145,22 @@ def _save_skip_storage(file, **kwargs) -> None:
 @doc_args(Artifact.path.__doc__)
 def path(self) -> Path | UPath:
     """{}"""  # noqa: D415
-    using_key = settings._using_key
-    return filepath_from_artifact(self, using_key)
+    # return only the path, without StorageSettings
+    filepath, _ = filepath_from_artifact(self, using_key=settings._using_key)
+    return filepath
+
+
+# get cache path without triggering sync
+@property  # type: ignore
+def _cache_path(self) -> UPath:
+    filepath, cache_key = filepath_cache_key_from_artifact(
+        self, using_key=settings._using_key
+    )
+    if isinstance(filepath, LocalPathClasses):
+        return filepath
+    return setup_settings.instance.storage.cloud_to_local_no_update(
+        filepath, cache_key=cache_key
+    )
 
 
 # docstring handled through attach_func_to_class_method
@@ -1173,6 +1199,7 @@ for name in METHOD_NAMES:
 # privates currently dealt with separately
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
+Artifact._cache_path = _cache_path
 Artifact.path = path
 Artifact.describe = describe
 Artifact.view_lineage = view_lineage

lamindb/_collection.py

@@ -211,6 +211,7 @@ def mapped(
     layers_keys: str | list[str] | None = None,
     obs_keys: str | list[str] | None = None,
     obsm_keys: str | list[str] | None = None,
+    obs_filter: tuple[str, str | tuple[str, ...]] | None = None,
     join: Literal["inner", "outer"] | None = "inner",
     encode_labels: bool | list[str] = True,
     unknown_label: str | dict[str, str] | None = None,
@@ -239,6 +240,7 @@ def mapped(
         layers_keys,
         obs_keys,
         obsm_keys,
+        obs_filter,
         join,
         encode_labels,
         unknown_label,

lamindb/_save.py

@@ -17,6 +17,7 @@ from lnschema_core.models import Artifact, Record
 
 from lamindb.core._settings import settings
 from lamindb.core.storage.paths import (
+    _cache_key_from_artifact_storage,
     attempt_accessing_path,
     auto_storage_key_from_artifact,
     delete_storage_using_key,
@@ -135,7 +136,7 @@ def check_and_attempt_upload(
     # a local env it will have a _local_filepath and needs to be uploaded
     if hasattr(artifact, "_local_filepath"):
         try:
-            storage_path = upload_artifact(
+            storage_path, cache_path = upload_artifact(
                 artifact,
                 using_key,
                 access_token=access_token,
@@ -146,7 +147,7 @@ def check_and_attempt_upload(
             return exception
         # copies (if on-disk) or moves the temporary file (if in-memory) to the cache
         if os.getenv("LAMINDB_MULTI_INSTANCE") is None:
-            copy_or_move_to_cache(artifact, storage_path)
+            copy_or_move_to_cache(artifact, storage_path, cache_path)
         # after successful upload, we should remove the attribute so that another call
         # call to save won't upload again, the user should call replace() then
         del artifact._local_filepath
@@ -154,7 +155,9 @@ def check_and_attempt_upload(
     return None
 
 
-def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
+def copy_or_move_to_cache(
+    artifact: Artifact, storage_path: UPath, cache_path: UPath | None
+):
     local_path = artifact._local_filepath
 
     # in-memory cases
@@ -166,7 +169,7 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
     cache_dir = settings._storage_settings.cache_dir
 
     # just delete from the cache dir if storage_path is local
-    if isinstance(storage_path, LocalPathClasses):
+    if cache_path is None:
         if (
             local_path.as_posix() != storage_path.as_posix()
             and cache_dir in local_path.parents
@@ -176,11 +179,12 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
             else:
                 local_path.unlink()
         return None
-
-    cache_path = settings._storage_settings.cloud_to_local_no_update(storage_path)
+    # non-local storage_path further
     if local_path != cache_path:
         cache_path.parent.mkdir(parents=True, exist_ok=True)
         if cache_dir in local_path.parents:
+            if cache_path.is_dir():
+                shutil.rmtree(cache_path)
             local_path.replace(cache_path)
         else:
             if is_dir:
@@ -280,11 +284,11 @@ def upload_artifact(
     using_key: str | None = None,
     access_token: str | None = None,
     print_progress: bool = True,
-) -> UPath:
+) -> tuple[UPath, UPath | None]:
     """Store and add file and its linked entries."""
     # can't currently use  filepath_from_artifact here because it resolves to ._local_filepath
     storage_key = auto_storage_key_from_artifact(artifact)
-    storage_path = attempt_accessing_path(
+    storage_path, storage_settings = attempt_accessing_path(
         artifact, storage_key, using_key=using_key, access_token=access_token
     )
     if hasattr(artifact, "_to_store") and artifact._to_store:
@@ -292,4 +296,13 @@ def upload_artifact(
         store_file_or_folder(
             artifact._local_filepath, storage_path, print_progress=print_progress
         )
-    return storage_path
+
+    if isinstance(storage_path, LocalPathClasses):
+        cache_path = None
+    else:
+        cache_key = _cache_key_from_artifact_storage(artifact, storage_settings)
+        cache_path = storage_settings.cloud_to_local_no_update(
+            storage_path, cache_key=cache_key
+        )
+
+    return storage_path, cache_path

lamindb/core/_context.py

@@ -450,11 +450,11 @@ class Context:
             self._logging_message += f"created Transform(uid='{transform.uid}')"
         else:
             uid = transform.uid
-            # check whether the transform file has been renamed
+            # check whether the transform.key is consistent
             if transform.key != key:
                 suid = transform.stem_uid
                 new_suid = ids.base62_12()
-                transform_type = "Notebook" if is_run_from_ipython else "Script"
+                transform_type = "notebook" if is_run_from_ipython else "script"
                 note = message_update_key_in_version_family(
                     suid=suid,
                     existing_key=transform.key,
@@ -462,7 +462,7 @@ class Context:
                     registry="Transform",
                 )
                 raise UpdateContext(
-                    f"{transform_type} filename changed.\n\nEither init a new transform family by setting:\n\n"
+                    f'\n✗ Filename "{key}" clashes with the existing key "{transform.key}" for uid "{transform.uid[:-4]}...."\n\nEither init a new transform with a new uid:\n\n'
                     f'ln.context.uid = "{new_suid}0000"\n\n{note}'
                 )
             elif transform.name != name:

lamindb/core/_data.py

@@ -41,7 +41,7 @@ if TYPE_CHECKING:
 
 
 WARNING_RUN_TRANSFORM = (
-    "no run & transform got linked, call `ln.context.track()` & re-run`"
+    "no run & transform got linked, call `ln.context.track()` & re-run"
 )
 
 WARNING_NO_INPUT = "run input wasn't tracked, call `ln.context.track()` and re-run"

lamindb/core/_mapped_collection.py

@@ -49,6 +49,9 @@ class _Connect:
             self.conn.close()
 
 
+_decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
+
+
 class MappedCollection:
     """Map-style collection for use in data loaders.
 
@@ -83,6 +86,9 @@ class MappedCollection:
             retrieves ``.X``.
         obsm_keys: Keys from the ``.obsm`` slots.
         obs_keys: Keys from the ``.obs`` slots.
+        obs_filter: Select only observations with these values for the given obs column.
+            Should be a tuple with an obs column name as the first element
+            and filtering values (a string or a tuple of strings) as the second element.
         join: `"inner"` or `"outer"` virtual joins. If ``None`` is passed,
             does not join.
         encode_labels: Encode labels into integers.
@@ -101,6 +107,7 @@ class MappedCollection:
         layers_keys: str | list[str] | None = None,
         obs_keys: str | list[str] | None = None,
         obsm_keys: str | list[str] | None = None,
+        obs_filter: tuple[str, str | tuple[str, ...]] | None = None,
         join: Literal["inner", "outer"] | None = "inner",
         encode_labels: bool | list[str] = True,
         unknown_label: str | dict[str, str] | None = None,
@@ -113,6 +120,13 @@ class MappedCollection:
                 f"join must be one of None, 'inner, or 'outer' but was {type(join)}"
             )
 
+        self.filtered = obs_filter is not None
+        if self.filtered and len(obs_filter) != 2:
+            raise ValueError(
+                "obs_filter should be a tuple with obs column name "
+                "as the first element and filtering values as the second element"
+            )
+
         if layers_keys is None:
             self.layers_keys = ["X"]
         else:
@@ -153,16 +167,37 @@ class MappedCollection:
         self.path_list = path_list
         self._make_connections(path_list, parallel)
 
+        self._cache_cats: dict = {}
+        if self.obs_keys is not None:
+            if cache_categories:
+                self._cache_categories(self.obs_keys)
+            self.encoders: dict = {}
+            if self.encode_labels:
+                self._make_encoders(self.encode_labels)  # type: ignore
+
         self.n_obs_list = []
+        self.indices_list = []
         for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
                 X = store["X"]
                 store_path = self.path_list[i]
                 self._check_csc_raise_error(X, "X", store_path)
-                if isinstance(X, ArrayTypes):  # type: ignore
-                    self.n_obs_list.append(X.shape[0])
+                if self.filtered:
+                    obs_filter_key, obs_filter_values = obs_filter
+                    indices_storage = np.where(
+                        np.isin(
+                            self._get_labels(store, obs_filter_key), obs_filter_values
+                        )
+                    )[0]
+                    n_obs_storage = len(indices_storage)
                 else:
-                    self.n_obs_list.append(X.attrs["shape"][0])
+                    if isinstance(X, ArrayTypes):  # type: ignore
+                        n_obs_storage = X.shape[0]
+                    else:
+                        n_obs_storage = X.attrs["shape"][0]
+                    indices_storage = np.arange(n_obs_storage)
+                self.n_obs_list.append(n_obs_storage)
+                self.indices_list.append(indices_storage)
                 for layer_key in self.layers_keys:
                     if layer_key == "X":
                         continue
@@ -180,7 +215,7 @@ class MappedCollection:
                         )
         self.n_obs = sum(self.n_obs_list)
 
-        self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
+        self.indices = np.hstack(self.indices_list)
         self.storage_idx = np.repeat(np.arange(len(self.storages)), self.n_obs_list)
 
         self.join_vars: Literal["inner", "outer"] | None = join
@@ -193,15 +228,6 @@ class MappedCollection:
             self._make_join_vars()
             self.n_vars = len(self.var_joint)
 
-        if self.obs_keys is not None:
-            if cache_categories:
-                self._cache_categories(self.obs_keys)
-            else:
-                self._cache_cats: dict = {}
-            self.encoders: dict = {}
-            if self.encode_labels:
-                self._make_encoders(self.encode_labels)  # type: ignore
-
         self._dtype = dtype
         self._closed = False
 
@@ -220,14 +246,15 @@ class MappedCollection:
 
     def _cache_categories(self, obs_keys: list):
         self._cache_cats = {}
-        decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
         for label in obs_keys:
             self._cache_cats[label] = []
             for storage in self.storages:
                 with _Connect(storage) as store:
                     cats = self._get_categories(store, label)
                     if cats is not None:
-                        cats = decode(cats) if isinstance(cats[0], bytes) else cats[...]
+                        cats = (
+                            _decode(cats) if isinstance(cats[0], bytes) else cats[...]
+                        )
                     self._cache_cats[label].append(cats)
 
     def _make_encoders(self, encode_labels: list):
@@ -366,7 +393,7 @@ class MappedCollection:
 
     def _get_data_idx(
         self,
-        lazy_data: ArrayType | GroupType,  # type: ignore
+        lazy_data: ArrayType | GroupType,
         idx: int,
         join_vars: Literal["inner", "outer"] | None = None,
         var_idxs_join: list | None = None,
@@ -480,25 +507,17 @@ class MappedCollection:
     def get_merged_labels(self, label_key: str):
         """Get merged labels for `label_key` from all `.obs`."""
         labels_merge = []
-        decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
         for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
-                codes = self._get_codes(store, label_key)
-                labels = decode(codes) if isinstance(codes[0], bytes) else codes
-                if label_key in self._cache_cats:
-                    cats = self._cache_cats[label_key][i]
-                else:
-                    cats = self._get_categories(store, label_key)
-                if cats is not None:
-                    cats = decode(cats) if isinstance(cats[0], bytes) else cats
-                    labels = cats[labels]
+                labels = self._get_labels(store, label_key, storage_idx=i)
+                if self.filtered:
+                    labels = labels[self.indices_list[i]]
                 labels_merge.append(labels)
         return np.hstack(labels_merge)
 
     def get_merged_categories(self, label_key: str):
         """Get merged categories for `label_key` from all `.obs`."""
         cats_merge = set()
-        decode = np.frompyfunc(lambda x: x.decode("utf-8"), 1, 1)
         for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
                 if label_key in self._cache_cats:
@@ -506,15 +525,15 @@ class MappedCollection:
                 else:
                     cats = self._get_categories(store, label_key)
                 if cats is not None:
-                    cats = decode(cats) if isinstance(cats[0], bytes) else cats
+                    cats = _decode(cats) if isinstance(cats[0], bytes) else cats
                     cats_merge.update(cats)
                 else:
                     codes = self._get_codes(store, label_key)
-                    codes = decode(codes) if isinstance(codes[0], bytes) else codes
+                    codes = _decode(codes) if isinstance(codes[0], bytes) else codes
                     cats_merge.update(codes)
         return sorted(cats_merge)
 
-    def _get_categories(self, storage: StorageType, label_key: str):  # type: ignore
+    def _get_categories(self, storage: StorageType, label_key: str):
         """Get categories."""
         obs = storage["obs"]  # type: ignore
         if isinstance(obs, ArrayTypes):  # type: ignore
@@ -543,7 +562,7 @@ class MappedCollection:
                     return None
         return None
 
-    def _get_codes(self, storage: StorageType, label_key: str):  # type: ignore
+    def _get_codes(self, storage: StorageType, label_key: str):
         """Get codes."""
         obs = storage["obs"]  # type: ignore
         if isinstance(obs, ArrayTypes):  # type: ignore
@@ -555,6 +574,21 @@ class MappedCollection:
             else:
                 return label["codes"][...]
 
+    def _get_labels(
+        self, storage: StorageType, label_key: str, storage_idx: int | None = None
+    ):
+        """Get labels."""
+        codes = self._get_codes(storage, label_key)
+        labels = _decode(codes) if isinstance(codes[0], bytes) else codes
+        if storage_idx is not None and label_key in self._cache_cats:
+            cats = self._cache_cats[label_key][storage_idx]
+        else:
+            cats = self._get_categories(storage, label_key)
+        if cats is not None:
+            cats = _decode(cats) if isinstance(cats[0], bytes) else cats
+            labels = cats[labels]
+        return labels
+
     def close(self):
         """Close connections to array streaming backend.
 

lamindb/core/datasets/_core.py

@@ -137,7 +137,17 @@ def file_tiff_suo22() -> Path:  # pragma: no cover
 def dir_iris_images() -> UPath:  # pragma: no cover
     """Directory with 3 studies of the Iris flower: 405 images & metadata.
 
-    Based on: https://github.com/laminlabs/lamindb-dev-datasets/pull/2
+    Provenance: https://lamin.ai/laminlabs/lamindata/transform/3q4MpQxRL2qZ5zKv
+
+    The problem is that the same artifact was also ingested by the downstream
+    demo notebook:
+    https://lamin.ai/laminlabs/lamindata/transform/NJvdsWWbJlZS5zKv
+
+    This is why on the UI, the artifact shows up as output of the downstream
+    demo notebook rather than the upstream curation notebook. The lineage
+    information should still be captured by
+    https://github.com/laminlabs/lnschema-core/blob/a90437e91dfbd6b9002f18c3e978bd0f9c9a632d/lnschema_core/models.py#L2050-L2052
+    but we don't use this in the UI yet.
     """
     return UPath("s3://lamindata/iris_studies")
 

lamindb/core/exceptions.py

@@ -1,10 +1,9 @@
 """Exceptions.
 
-The registry base class:
-
 .. autosummary::
    :toctree: .
 
+   InvalidArgument
    DoesNotExist
    ValidationError
    NotebookNotSavedError
@@ -15,12 +14,26 @@ The registry base class:
 
 """
 
+# inheriting from SystemExit has the sole purpose of suppressing
+# the traceback - this isn't optimal but the current best solution
+# https://laminlabs.slack.com/archives/C04A0RMA0SC/p1726856875597489
+
+
+class InvalidArgument(SystemExit):
+    """Invalid method or function argument."""
+
+    pass
+
 
 class TrackNotCalled(SystemExit):
+    """ln.context.track() wasn't called."""
+
     pass
 
 
 class NotebookNotSaved(SystemExit):
+    """Notebook wasn't saved."""
+
     pass
 
 

lamindb/core/storage/_backed_access.py

@@ -69,7 +69,7 @@ def backed_access(
     using_key: str | None = None,
 ) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
     if isinstance(artifact_or_filepath, Artifact):
-        filepath = filepath_from_artifact(artifact_or_filepath, using_key=using_key)
+        filepath, _ = filepath_from_artifact(artifact_or_filepath, using_key=using_key)
     else:
         filepath = artifact_or_filepath
     name = filepath.name

lamindb/core/storage/paths.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import shutil
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 import anndata as ad
@@ -19,6 +18,8 @@ from lnschema_core.models import Artifact, Storage
 from lamindb.core._settings import settings
 
 if TYPE_CHECKING:
+    from pathlib import Path
+
     from lamindb_setup.core.types import UPathStr
 
 
@@ -52,12 +53,13 @@ def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -
     return root.resolve() in path.resolve().parents
 
 
+# returns filepath and root of the storage
 def attempt_accessing_path(
     artifact: Artifact,
     storage_key: str,
     using_key: str | None = None,
     access_token: str | None = None,
-):
+) -> tuple[UPath, StorageSettings]:
     # check whether the file is in the default db and whether storage
     # matches default storage
     if (
@@ -78,23 +80,53 @@ def attempt_accessing_path(
         # find a better way than passing None to instance_settings in the future!
         storage_settings = StorageSettings(storage.root, access_token=access_token)
     path = storage_settings.key_to_filepath(storage_key)
-    return path
+    return path, storage_settings
 
 
-# add type annotations back asap when re-organizing the module
-def filepath_from_artifact(artifact: Artifact, using_key: str | None = None):
+def filepath_from_artifact(
+    artifact: Artifact, using_key: str | None = None
+) -> tuple[UPath, StorageSettings | None]:
     if hasattr(artifact, "_local_filepath") and artifact._local_filepath is not None:
-        return artifact._local_filepath.resolve()
+        return artifact._local_filepath.resolve(), None
     storage_key = auto_storage_key_from_artifact(artifact)
-    path = attempt_accessing_path(artifact, storage_key, using_key=using_key)
-    return path
+    path, storage_settings = attempt_accessing_path(
+        artifact, storage_key, using_key=using_key
+    )
+    return path, storage_settings
+
+
+# virtual key is taken into consideration
+# only if the version is latest
+def _cache_key_from_artifact_storage(
+    artifact: Artifact, storage_settings: StorageSettings | None
+):
+    cache_key = None
+    if (
+        artifact._key_is_virtual
+        and artifact.key is not None
+        and storage_settings is not None
+        and artifact.is_latest
+    ):
+        cache_key = (storage_settings.root / artifact.key).path
+    return cache_key
+
+
+# return filepath and cache_key if needed
+def filepath_cache_key_from_artifact(
+    artifact: Artifact, using_key: str | None = None
+) -> tuple[UPath, str | None]:
+    filepath, storage_settings = filepath_from_artifact(artifact, using_key)
+    if isinstance(filepath, LocalPathClasses):
+        return filepath, None
+    cache_key = _cache_key_from_artifact_storage(artifact, storage_settings)
+    return filepath, cache_key
 
 
 def store_file_or_folder(
     local_path: UPathStr, storage_path: UPath, print_progress: bool = True
 ) -> None:
     """Store file or folder (localpath) at storagepath."""
-    local_path = Path(local_path)
+    local_path = UPath(local_path)
     if not isinstance(storage_path, LocalPathClasses):
         # this uploads files and directories
         create_folder = False if local_path.is_dir() else None
@@ -102,12 +134,11 @@ def store_file_or_folder(
             local_path, create_folder=create_folder, print_progress=print_progress
         )
     else:  # storage path is local
+        if local_path.resolve().as_posix() == storage_path.resolve().as_posix():
+            return None
         storage_path.parent.mkdir(parents=True, exist_ok=True)
         if local_path.is_file():
-            try:
-                shutil.copyfile(local_path, storage_path)
-            except shutil.SameFileError:
-                pass
+            shutil.copyfile(local_path, storage_path)
         else:
             if storage_path.exists():
                 shutil.rmtree(storage_path)
@@ -117,7 +148,7 @@ def store_file_or_folder(
 def delete_storage_using_key(
     artifact: Artifact, storage_key: str, using_key: str | None
 ):
-    filepath = attempt_accessing_path(artifact, storage_key, using_key=using_key)
+    filepath, _ = attempt_accessing_path(artifact, storage_key, using_key=using_key)
     delete_storage(filepath)
 
 

lamindb/core/versioning.py

@@ -18,7 +18,7 @@ def message_update_key_in_version_family(
     registry: str,
     new_key: str,
 ) -> str:
-    return f'Or update key "{existing_key}" in your existing family:\n\nln.{registry}.filter(uid__startswith="{suid}").update(key="{new_key}")'
+    return f'Or update key "{existing_key}" to "{new_key}":\n\nln.{registry}.filter(uid__startswith="{suid}").update(key="{new_key}")\n'
 
 
 def increment_base62(s: str) -> str:

{lamindb-0.76.7.dist-info → lamindb-0.76.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.7
+Version: 0.76.9
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.74.3
-Requires-Dist: lamindb_setup==0.77.2
+Requires-Dist: lnschema_core==0.74.5
+Requires-Dist: lamindb_setup==0.77.5
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.17.3
+Requires-Dist: lamin_cli==0.17.6
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0

{lamindb-0.76.7.dist-info → lamindb-0.76.9.dist-info}/RECORD

@@ -1,7 +1,7 @@
-lamindb/__init__.py,sha256=5ywGXz6u0OjQ-W57dmRORaBl-njA98gRYHBZgW9HBh8,2344
-lamindb/_artifact.py,sha256=PKu_CuTypCSSByu0bjRELVWsZ6mQU-AFIygGAAXVsIs,44110
+lamindb/__init__.py,sha256=2tV0Xvd0v3bD22wAZj__CvVleGCC1UG7zEqm1Nkb1q8,2344
+lamindb/_artifact.py,sha256=ZQ5tFPhwnC9Aqku-wPBflXU_u8jh6EroCnZPsZE_42M,44984
 lamindb/_can_validate.py,sha256=9di9FLmC2m3dpT42sceF34UEFzQITi2e_hjVMa8DIc4,18261
-lamindb/_collection.py,sha256=hT9VFNIVd041f45bDE-BYXcCvM2Cd1vGByTeP9_F3Yc,14016
+lamindb/_collection.py,sha256=Lnmc42wQgN0-qrPBpGaPPt_rDtYgOBjYVhAkuZNS4uQ,14101
 lamindb/_curate.py,sha256=b9CsOChho-V9YUOY2D4ZO-agM0wjH5DsBGxcrRCJuTw,58807
 lamindb/_feature.py,sha256=nZhtrH0ssoNls-hV-dkwfK9sKypg2El59R9qfarxfUE,5340
 lamindb/_feature_set.py,sha256=DmAy96V_RyV0yiyvWOCHgustXPsCaMwn4TrWwh2qDd8,8104
@@ -14,39 +14,39 @@ lamindb/_query_manager.py,sha256=Ipe85HL31DDwMbC8CN_1Svbwk48a_DUh_INGQdZL08I,422
 lamindb/_query_set.py,sha256=BiGvEiaBSd9aV28EAy83Q8h6RLsYMDjfxLOljAcyMaM,12692
 lamindb/_record.py,sha256=d-tBYj_EgNBD2Nl9pBC5H_OYGdRmBKAVvRnE91_23e8,22035
 lamindb/_run.py,sha256=5M_r1zGDv9HlqbqRKTWCYCOtENovJ-8mQ4kY7XqcLaU,1888
-lamindb/_save.py,sha256=Fu7Z84btKOXfTfpunKLni21s5ER2zIllqg5e3nPq-0A,10910
+lamindb/_save.py,sha256=YfyTTOil-05OBfs5bO1vFxuyUYUqR5J1wqxaKYZxg4s,11352
 lamindb/_storage.py,sha256=GBVChv-DHVMNEBJL5l_JT6B4RDhZ6NnwgzmUICphYKk,413
 lamindb/_transform.py,sha256=tRO7Uq-8fkq6Tm4U5qQ1lBOaNUehH8IkiDDPnYPgQH8,4623
 lamindb/_ulabel.py,sha256=XDSdZBXX_ki5s1vOths3MjF2x5DPggBR_PV_KF4SGyg,1611
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=4Ln2ItTb3857PAI-70O8eJYqoTJ_NNFc7E_wds6OGns,2412
 lamindb/core/__init__.py,sha256=57AXQ286eOX2_o5HUeqIFJrfqN-OZ_E7FVHd3Xm5oOk,1483
-lamindb/core/_context.py,sha256=RCp5NKNr8JWxHZXFmNZT0GYgt-nKs4inZ0OQYkBMub0,21549
-lamindb/core/_data.py,sha256=EJNVqsyBR5N-IK-YtF6l92QlB2V9FFRkkd4uF_Phfmo,16890
+lamindb/core/_context.py,sha256=O66uBJGDHBJFCKFrQyFuBPheCxXMSMmDjelIjLpfiSQ,21613
+lamindb/core/_data.py,sha256=TRgB_VFXjToVeFkqlZLROvdAcl2-2jg9QJLGQRPf8i0,16889
 lamindb/core/_feature_manager.py,sha256=n8z51HiOGom642EXmSrHMDqlcpbJFyYutyc-Mwu1NM0,32673
 lamindb/core/_label_manager.py,sha256=VskOVnQoafKCjUKDs64b3vN4w7NTcM7fDXNdUtMVGFo,9605
-lamindb/core/_mapped_collection.py,sha256=1XzratL2IvRleqioNhWo26Lsuqkev8-HEImmHQxw9Kw,23266
+lamindb/core/_mapped_collection.py,sha256=h8H1j_TC_Tzk2CHcMjTRk34Et95SVQY5di4GnQdmqxc,24604
 lamindb/core/_settings.py,sha256=73SV-vTDzSKX9E5rSvj9kdPV4jHSovRM3x7bgT1OBh8,5948
 lamindb/core/_sync_git.py,sha256=qc0yfPyKeG4uuNT_3qsv-mkIMqhLFqfXNeNVO49vV00,4547
 lamindb/core/_track_environment.py,sha256=STzEVUzOeUEWdX7WDJUkKH4u08k7eupRX6AXQwoVt14,828
-lamindb/core/exceptions.py,sha256=9KM3j2PvHzW-Gx6waoxzlYiz822ZMJ_7PXPqv1AHup0,1284
+lamindb/core/exceptions.py,sha256=rJ7v4KZWBrYP62ylIy5jCRvCW0CteQ-4r4yMrUz2UP8,1652
 lamindb/core/fields.py,sha256=47Jmh3efUr5ZscgimR_yckY-I3cNf8ScLutbwKCK3j4,162
 lamindb/core/loaders.py,sha256=KMTkDa73jkRVvI9uc5Fgr0t6mq22cAxBwhSlUZKUaBg,4016
 lamindb/core/schema.py,sha256=KiYQn_8fokSMztTNDe6qUocZzKXWxU32H-YChNJv51A,1877
 lamindb/core/types.py,sha256=uVBqSVLoQaTkqP9nqsJhwU6yYnx8H5e6-ZxrB6vpOOw,265
-lamindb/core/versioning.py,sha256=GYhgSA6IOlWMMNfctZu7U_jIvmQP2gdvsZxn4bTanOc,5277
+lamindb/core/versioning.py,sha256=i9J6YaibM4dXqojqfC_tULfEAJXHu6hU3XbsIeKgp_4,5270
 lamindb/core/datasets/__init__.py,sha256=zRP98oqUAaXhqWyKMiH0s_ImVIuNeziQQ2kQ_t0f-DI,1353
-lamindb/core/datasets/_core.py,sha256=CgVF_pXuBXLElzubDMsl1DbpYOnXCY0HleITVvBKih4,19873
+lamindb/core/datasets/_core.py,sha256=JGP_q-OQibDCEaI54jZ2F6fSbSW9Yg6oYOqgOCXM0v4,20414
 lamindb/core/datasets/_fake.py,sha256=BZF9R_1iF0HDnvtZNqL2FtsjSMuqDIfuFxnw_LJYIh4,953
 lamindb/core/storage/__init__.py,sha256=JOIMu_7unbyhndtH1j0Q-9AvY8knSuc1IJO9sQnyBAQ,498
 lamindb/core/storage/_anndata_accessor.py,sha256=F3ze8ICG7K4BKueg-766olnoEA8Eh8gVrvDSSE2FX-M,24160
 lamindb/core/storage/_anndata_sizes.py,sha256=aXO3OB--tF5MChenSsigW6Q-RuE8YJJOUTVukkLrv9A,1029
-lamindb/core/storage/_backed_access.py,sha256=YcWCeT2eligJGsBdjJS_-4el_eC9J088jxUWG9lsleM,3231
+lamindb/core/storage/_backed_access.py,sha256=O0zazsDlW0PKa52WHV5HooHGGI81FxLT6VEvNONdiEc,3234
 lamindb/core/storage/_tiledbsoma.py,sha256=0NPLS5m1icEhzWPfXAv4U2SNiLGqGQd7FM6xCm5wYEc,7269
 lamindb/core/storage/_valid_suffixes.py,sha256=vUSeQ4s01rdhD_vSd6wKmFBsgMJAKkBMnL_T9Y1znMg,501
 lamindb/core/storage/_zarr.py,sha256=TODQD3p1eykoPwP-c-YRP_UDmsbMeBGMGvkBxxOMeYc,3663
 lamindb/core/storage/objects.py,sha256=OzvBCS-Urz5mr-O95qYt6RGBDDX5HmjfRRKWPPDn1ZE,1797
-lamindb/core/storage/paths.py,sha256=L5ImdOURHdA9dB2XLzuFe90zj2oC1EQaZdi0pyYZcW0,4854
+lamindb/core/storage/paths.py,sha256=bQwRbGTOCx3DSFOQ404uT-3YBXrm8yfKuttNCdWwJpA,5892
 lamindb/core/subsettings/__init__.py,sha256=KFHPzIE7f7Bj4RgMjGQF4CjTdHVG_VNFBrCndo49ixo,198
 lamindb/core/subsettings/_creation_settings.py,sha256=54mfMH_osC753hpxcl7Dq1rwBD2LHnWveXtQpkLBITE,1194
 lamindb/core/subsettings/_transform_settings.py,sha256=4YbCuZtJo6zdytl6UQR4GvdDkTtT6SRBqVzofGzNOt8,583
@@ -54,7 +54,7 @@ lamindb/integrations/__init__.py,sha256=RWGMYYIzr8zvmNPyVB4m-p4gMDhxdRbjES2Ed23O
 lamindb/integrations/_vitessce.py,sha256=S51wl7iF2QvQmrNcZ9yDdqTtcn_AAzuh0i5axKwQ2sM,4560
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.76.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.76.7.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.76.7.dist-info/METADATA,sha256=8F_urb2kwsRFh0LjMICrKOS6AwoBy7GTD837GPsm9pA,2372
-lamindb-0.76.7.dist-info/RECORD,,
+lamindb-0.76.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.76.9.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.76.9.dist-info/METADATA,sha256=lHl2Ifd3QCisYZ-Qz6JacQMWvXtF6O7HQfK65JiKBb4,2372
+lamindb-0.76.9.dist-info/RECORD,,