lamindb 0.69.10__py3-none-any.whl → 0.70.1__py3-none-any.whl

lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.10"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

lamindb/_artifact.py

@@ -37,7 +37,7 @@ from lamindb.core.storage import (
     size_adata,
     write_to_file,
 )
-from lamindb.core.storage.file import (
+from lamindb.core.storage.paths import (
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     filepath_from_artifact,
@@ -50,8 +50,8 @@ from .core._data import (
     save_feature_set_links,
     save_feature_sets,
 )
-from .core.storage.file import AUTO_KEY_PREFIX
-from .core.storage.object import _mudata_is_installed
+from .core.storage.objects import _mudata_is_installed
+from .core.storage.paths import AUTO_KEY_PREFIX
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
@@ -167,7 +167,7 @@ def process_data(
         # Alex: I don't understand the line below
         if path.suffixes == []:
             path = path.with_suffix(suffix)
-        if suffix not in {".zarr", ".zrad"}:
+        if suffix != ".zarr":
             write_to_file(data, path)
         use_existing_storage_key = False
     else:
@@ -188,11 +188,7 @@ def get_stat_or_artifact(
     n_objects = None
     if settings.upon_file_create_skip_size_hash:
         return None, None, None, n_objects
-    if (
-        suffix in {".zarr", ".zrad"}
-        and memory_rep is not None
-        and isinstance(memory_rep, AnnData)
-    ):
+    if suffix == ".zarr" and memory_rep is not None and isinstance(memory_rep, AnnData):
         size = size_adata(memory_rep)
         return size, None, None, n_objects
     stat = path.stat()  # one network request
@@ -238,7 +234,7 @@ def get_stat_or_artifact(
                 "💡 you can make this error a warning:\n"
                 "    ln.settings.upon_artifact_create_if_hash_exists"
             )
-            raise RuntimeError(f"{msg}\n{hint}")
+            raise FileExistsError(f"{msg}\n{hint}")
         elif settings.upon_artifact_create_if_hash_exists == "warn_create_new":
             logger.warning(
                 "creating new Artifact object despite existing artifact with same hash:"
@@ -246,10 +242,12 @@ def get_stat_or_artifact(
             )
             return size, hash, hash_type, n_objects
         else:
-            from_trash = "(from trash)" if result[0].visibility == -1 else ""
-            logger.warning(
-                f"returning existing artifact with same hash{from_trash}: {result[0]}"
-            )
+            if result[0].visibility == -1:
+                raise FileExistsError(
+                    f"You're trying to re-create this artifact in trash: {result[0]}"
+                    "Either permanently delete it with `artifact.delete(permanent=True)` or restore it with `artifact.restore()`"
+                )
+            logger.warning(f"returning existing artifact with same hash: {result[0]}")
             return result[0]
     else:
         return size, hash, hash_type, n_objects
@@ -331,6 +329,15 @@ def get_artifact_kwargs_from_data(
         using_key=using_key,
     )
     if isinstance(stat_or_artifact, Artifact):
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if stat_or_artifact.run is not None:
+                stat_or_artifact.run.replicated_output_artifacts.add(stat_or_artifact)
+            # update the run of the artifact with the latest run
+            stat_or_artifact.run = run
+            stat_or_artifact.transform = run.transform
         return stat_or_artifact, None
     else:
         size, hash, hash_type, n_objects = stat_or_artifact
@@ -439,7 +446,12 @@ def data_is_anndata(data: AnnData | UPathStr):
     if isinstance(data, AnnData):
         return True
     if isinstance(data, (str, Path, UPath)):
-        return Path(data).suffix in {".h5ad", ".zrad"}
+        if Path(data).suffix == ".h5ad":
+            return True
+        elif Path(data).suffix == ".zarr":
+            raise NotImplementedError(
+                "auto-detecting AnnData from Zarr is not yet supported"
+            )
     return False
 
 
@@ -821,7 +833,7 @@ def replace(
 
 # docstring handled through attach_func_to_class_method
 def backed(self, is_run_input: bool | None = None) -> AnnDataAccessor | BackedAccessor:
-    suffixes = (".h5", ".hdf5", ".h5ad", ".zrad", ".zarr")
+    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr")
     if self.suffix not in suffixes:
         raise ValueError(
             "Artifact should have a zarr or h5 object as the underlying data, please"
@@ -854,7 +866,7 @@ def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs)
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> Path:
+def cache(self, is_run_input: bool | None = None) -> Path:
     _track_run_input(self, is_run_input)
 
     using_key = settings._using_key
@@ -932,10 +944,10 @@ def _delete_skip_storage(artifact, *args, **kwargs) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, *args, **kwargs) -> None:
+def save(self, upload: bool | None = None, **kwargs) -> None:
     access_token = kwargs.pop("access_token", None)
 
-    self._save_skip_storage(*args, **kwargs)
+    self._save_skip_storage(**kwargs)
 
     from lamindb._save import check_and_attempt_clearing, check_and_attempt_upload
 
@@ -951,9 +963,9 @@ def save(self, *args, **kwargs) -> None:
         raise RuntimeError(exception)
 
 
-def _save_skip_storage(file, *args, **kwargs) -> None:
+def _save_skip_storage(file, **kwargs) -> None:
     save_feature_sets(file)
-    super(Artifact, file).save(*args, **kwargs)
+    super(Artifact, file).save(**kwargs)
     save_feature_set_links(file)
 
 
@@ -998,7 +1010,7 @@ METHOD_NAMES = [
     "from_df",
     "from_mudata",
     "backed",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -1024,5 +1036,6 @@ for name in METHOD_NAMES:
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
+Artifact.stage = cache
 # this seems a Django-generated function
 delattr(Artifact, "get_visibility_display")

lamindb/_can_validate.py

@@ -30,6 +30,7 @@ def inspect(
     *,
     mute: bool = False,
     organism: str | Registry | None = None,
+    public_source: Registry | None = None,
 ) -> InspectResult:
     """{}."""
     return _inspect(
@@ -38,6 +39,7 @@ def inspect(
         field=field,
         mute=mute,
         organism=organism,
+        public_source=public_source,
     )
 
 
@@ -63,6 +65,7 @@ def _inspect(
     mute: bool = False,
     using_key: str | None = None,
     organism: str | Registry | None = None,
+    public_source: Registry | None = None,
 ) -> pd.DataFrame | dict[str, list[str]]:
     """{}."""
     from lamin_utils._inspect import inspect
@@ -86,9 +89,9 @@ def _inspect(
 
     if len(nonval) > 0 and orm.__get_schema_name__() == "bionty":
         try:
-            bionty_result = orm.public(organism=organism).inspect(
-                values=nonval, field=field, mute=True
-            )
+            bionty_result = orm.public(
+                organism=organism, public_source=public_source
+            ).inspect(values=nonval, field=field, mute=True)
             bionty_validated = bionty_result.validated
             bionty_mapper = bionty_result.synonyms_mapper
             hint = False

lamindb/_collection.py

@@ -116,6 +116,17 @@ def __init__(
         logger.warning(
             f"returning existing collection with same hash: {existing_collection}"
         )
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if existing_collection.run is not None:
+                existing_collection.run.replicated_output_collections.add(
+                    existing_collection
+                )
+            # update the run of the artifact with the latest run
+            existing_collection.run = run
+            existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         for slot, feature_set in collection.features._feature_set_by_slot.items():
             if slot in feature_sets:
@@ -223,11 +234,11 @@ def mapped(
 ) -> MappedCollection:
     path_list = []
     for artifact in self.artifacts.all():
-        if artifact.suffix not in {".h5ad", ".zrad", ".zarr"}:
+        if artifact.suffix not in {".h5ad", ".zarr"}:
             logger.warning(f"Ignoring artifact with suffix {artifact.suffix}")
             continue
         elif not stream:
-            path_list.append(artifact.stage())
+            path_list.append(artifact.cache())
         else:
             path_list.append(artifact.path)
     ds = MappedCollection(
@@ -248,11 +259,11 @@ def mapped(
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> list[UPath]:
+def cache(self, is_run_input: bool | None = None) -> list[UPath]:
     _track_run_input(self, is_run_input)
     path_list = []
     for artifact in self.artifacts.all():
-        path_list.append(artifact.stage())
+        path_list.append(artifact.cache())
     return path_list
 
 
@@ -355,7 +366,7 @@ def artifacts(self) -> QuerySet:
 METHOD_NAMES = [
     "__init__",
     "mapped",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -377,3 +388,4 @@ for name in METHOD_NAMES:
 # this seems a Django-generated function
 delattr(Collection, "get_visibility_display")
 Collection.artifacts = artifacts
+Collection.stage = cache

lamindb/_finish.py

@@ -43,39 +43,35 @@ def finish(i_saved_the_notebook: bool = False):
                 "Please pass `i_saved_the_notebook=True` to `ln.finish()`, save the notebook, and re-run this cell."
             )
             return None
-        notebook_content = read_notebook(run_context.path)  # type: ignore
-        if not check_last_cell(notebook_content, "i_saved_the_notebook"):
-            raise CallFinishInLastCell(
-                "Can only run `ln.finish(i_saved_the_notebook=True)` from the last code cell of the notebook."
-            )
         save_run_context_core(
             run=run_context.run,
             transform=run_context.transform,
             filepath=run_context.path,
             finished_at=True,
-            notebook_content=notebook_content,
         )
     else:
         # scripts
+        # save_run_context_core was already called during ln.track()
         run_context.run.finished_at = datetime.now(timezone.utc)  # update run time
         run_context.run.save()
 
 
-# do not type because we need to be aware of lnschema_core import order
 def save_run_context_core(
     *,
     run: Run,
     transform: Transform,
     filepath: Path,
     transform_family: QuerySet | None = None,
-    is_consecutive: bool = True,
     finished_at: bool = False,
-    notebook_content=None,  # nbproject.Notebook
 ) -> str | None:
     import lamindb as ln
 
     ln.settings.verbosity = "success"
 
+    # for scripts, things are easy
+    is_consecutive = True
+    source_code_path = filepath
+    # for notebooks, we need more work
     if transform.type == TransformType.notebook:
         try:
             import nbstripout
@@ -88,62 +84,52 @@ def save_run_context_core(
                 "install nbproject & nbstripout: pip install nbproject nbstripout"
             )
             return None
-        if notebook_content is None:
-            notebook_content = read_notebook(filepath)  # type: ignore
+        notebook_content = read_notebook(filepath)  # type: ignore
         is_consecutive = check_consecutiveness(notebook_content)
         if not is_consecutive:
+            msg = "   Do you still want to proceed with finishing? (y/n) "
             if os.getenv("LAMIN_TESTING") is None:
-                decide = input(
-                    "   Do you still want to proceed with publishing? (y/n) "
-                )
+                response = input(msg)
             else:
-                decide = "n"
-            if decide != "y":
-                logger.error("Aborted (non-consecutive)!")
+                response = "n"
+            if response != "y":
                 return "aborted-non-consecutive"
-
         # convert the notebook file to html
         # log_level is set to 40 to silence the nbconvert logging
-        result = subprocess.run(
+        subprocess.run(
             "jupyter nbconvert --to html"
             f" {filepath.as_posix()} --Application.log_level=40",
             shell=True,
+            check=True,
         )
         # move the temporary file into the cache dir in case it's accidentally
         # in an existing storage location -> we want to move associated
         # artifacts into default storage and not register them in an existing
         # location
-        filepath_html = filepath.with_suffix(".html")  # current location
+        filepath_html_orig = filepath.with_suffix(".html")  # current location
+        filepath_html = ln_setup.settings.storage.cache_dir / filepath_html_orig.name
+        # don't use Path.rename here because of cross-device link error
+        # https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
         shutil.move(
-            filepath_html,  # type: ignore
-            ln_setup.settings.storage.cache_dir / filepath_html.name,
-        )  # move; don't use Path.rename here because of cross-device link error
-        # see https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
-        filepath_html = (
-            ln_setup.settings.storage.cache_dir / filepath_html.name
-        )  # adjust location
-        assert result.returncode == 0
-        # copy the notebook file to a temporary file
+            filepath_html_orig,  # type: ignore
+            filepath_html,
+        )
+        # strip the output from the notebook to create the source code file
+        # first, copy the notebook file to a temporary file in the cache
         source_code_path = ln_setup.settings.storage.cache_dir / filepath.name
         shutil.copy2(filepath, source_code_path)  # copy
-        result = subprocess.run(f"nbstripout {source_code_path}", shell=True)
-        assert result.returncode == 0
-    else:
-        source_code_path = filepath
+        subprocess.run(f"nbstripout {source_code_path}", shell=True, check=True)
     # find initial versions of source codes and html reports
-    initial_report = None
-    initial_source = None
+    prev_report = None
+    prev_source = None
     if transform_family is None:
         transform_family = transform.versions
     if len(transform_family) > 0:
         for prev_transform in transform_family.order_by("-created_at"):
-            # check for id to avoid query
             if prev_transform.latest_report_id is not None:
-                # any previous latest report of this transform is OK!
-                initial_report = prev_transform.latest_report
+                prev_report = prev_transform.latest_report
             if prev_transform.source_code_id is not None:
-                # any previous source code id is OK!
-                initial_source = prev_transform.source_code
+                prev_source = prev_transform.source_code
     ln.settings.silence_file_run_transform_warning = True
     # register the source code
     if transform.source_code is not None:
@@ -173,7 +159,7 @@ def save_run_context_core(
             source_code_path,
             description=f"Source of transform {transform.uid}",
             version=transform.version,
-            is_new_version_of=initial_source,
+            is_new_version_of=prev_source,
             visibility=0,  # hidden file
             run=False,
         )
@@ -207,7 +193,7 @@ def save_run_context_core(
             report_file = ln.Artifact(
                 filepath_html,
                 description=f"Report of run {run.uid}",
-                is_new_version_of=initial_report,
+                is_new_version_of=prev_report,
                 visibility=0,  # hidden file
                 run=False,
             )

lamindb/_from_values.py

@@ -21,6 +21,7 @@ def get_or_create_records(
     from_public: bool = False,
     organism: Registry | str | None = None,
     public_source: Registry | None = None,
+    mute: bool = False,
 ) -> list[Registry]:
     """Get or create records from iterables."""
     upon_create_search_names = settings.upon_create_search_names
@@ -38,14 +39,18 @@ def get_or_create_records(
 
         # returns existing records & non-existing values
         records, nonexist_values, msg = get_existing_records(
-            iterable_idx=iterable_idx, field=field, **kwargs
+            iterable_idx=iterable_idx, field=field, mute=mute, **kwargs
         )
 
         # new records to be created based on new values
         if len(nonexist_values) > 0:
             if from_public:
                 records_bionty, unmapped_values = create_records_from_public(
-                    iterable_idx=nonexist_values, field=field, msg=msg, **kwargs
+                    iterable_idx=nonexist_values,
+                    field=field,
+                    msg=msg,
+                    mute=mute,
+                    **kwargs,
                 )
                 if len(records_bionty) > 0:
                     msg = ""
@@ -56,16 +61,17 @@ def get_or_create_records(
                 unmapped_values = nonexist_values
             # unmapped new_ids will NOT create records
             if len(unmapped_values) > 0:
-                if len(msg) > 0:
+                if len(msg) > 0 and not mute:
                     logger.success(msg)
                 s = "" if len(unmapped_values) == 1 else "s"
                 print_values = colors.yellow(_print_values(unmapped_values))
                 name = Registry.__name__
                 n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
-                logger.warning(
-                    f"{colors.red('did not create')} {name} record{s} for "
-                    f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
-                )
+                if not mute:
+                    logger.warning(
+                        f"{colors.red('did not create')} {name} record{s} for "
+                        f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
+                    )
         if Registry.__module__.startswith("lnschema_bionty.") or Registry == ULabel:
             if isinstance(iterable, pd.Series):
                 feature = iterable.name
@@ -85,6 +91,7 @@ def get_or_create_records(
 def get_existing_records(
     iterable_idx: pd.Index,
     field: StrField,
+    mute: bool = False,
     **kwargs,
 ):
     model = field.field.model
@@ -96,7 +103,11 @@ def get_existing_records(
     # standardize based on the DB reference
     # log synonyms mapped terms
     result = model.inspect(
-        iterable_idx, field=field, organism=kwargs.get("organism"), mute=True
+        iterable_idx,
+        field=field,
+        organism=kwargs.get("organism"),
+        public_source=kwargs.get("public_source"),
+        mute=True,
     )
     syn_mapper = result.synonyms_mapper
 
@@ -146,9 +157,10 @@ def get_existing_records(
     # no logging if all values are validated
     # logs if there are synonyms
     if len(syn_msg) > 0:
-        if len(msg) > 0:
+        if len(msg) > 0 and not mute:
             logger.success(msg)
-        logger.success(syn_msg)
+        if not mute:
+            logger.success(syn_msg)
         msg = ""
 
     existing_values = iterable_idx.intersection(
@@ -163,6 +175,7 @@ def create_records_from_public(
     iterable_idx: pd.Index,
     field: StrField,
     msg: str = "",
+    mute: bool = False,
     **kwargs,
 ):
     model = field.field.model
@@ -219,19 +232,20 @@ def create_records_from_public(
             s = "" if len(validated) == 1 else "s"
             print_values = colors.purple(_print_values(validated))
             # this is the success msg for existing records in the DB
-            if len(msg) > 0:
+            if len(msg) > 0 and not mute:
                 logger.success(msg)
-            logger.success(
-                "created"
-                f" {colors.purple(f'{len(validated)} {model.__name__} record{s} from Bionty')}"
-                f" matching {colors.italic(f'{field.field.name}')}: {print_values}"
-            )
+            if not mute:
+                logger.success(
+                    "created"
+                    f" {colors.purple(f'{len(validated)} {model.__name__} record{s} from Bionty')}"
+                    f" matching {colors.italic(f'{field.field.name}')}: {print_values}"
+                )
 
     # make sure that synonyms logging appears after the field logging
-    if len(msg_syn) > 0:
+    if len(msg_syn) > 0 and not mute:
         logger.success(msg_syn)
     # warning about multi matches
-    if len(multi_msg) > 0:
+    if len(multi_msg) > 0 and not mute:
         logger.warning(multi_msg)
 
     # return the values that are not found in the bionty reference
@@ -247,7 +261,8 @@ def index_iterable(iterable: Iterable) -> pd.Index:
 
 
 def _print_values(names: list, n: int = 20) -> str:
-    print_values = ", ".join([f"'{name}'" for name in names[:n]])
+    names = list(set(names))
+    print_values = ", ".join([f"'{name}'" for name in names[:n] if name != "None"])
     if len(names) > n:
         print_values += ", ..."
     return print_values

lamindb/_registry.py

@@ -134,6 +134,7 @@ def from_values(
     field: StrField | None = None,
     organism: Registry | str | None = None,
     public_source: Registry | None = None,
+    mute: bool = False,
 ) -> list[Registry]:
     """{}."""
     from_public = True if cls.__module__.startswith("lnschema_bionty.") else False
@@ -144,6 +145,7 @@ def from_values(
         from_public=from_public,
         organism=organism,
         public_source=public_source,
+        mute=mute,
     )
 
 

lamindb/_save.py

@@ -16,11 +16,11 @@ from lamindb_setup.core.upath import UPath, print_hook
 from lnschema_core.models import Artifact, Registry
 
 from lamindb.core._settings import settings
-from lamindb.core.storage.file import (
+from lamindb.core.storage.paths import (
     attempt_accessing_path,
     auto_storage_key_from_artifact,
     delete_storage_using_key,
-    store_artifact,
+    store_file_or_folder,
 )
 
 try:
@@ -286,7 +286,7 @@ def upload_artifact(
     )
     msg = f"storing artifact '{artifact.uid}' at '{storage_path}'"
     if (
-        artifact.suffix in {".zarr", ".zrad"}
+        artifact.suffix == ".zarr"
         and hasattr(artifact, "_memory_rep")
         and artifact._memory_rep is not None
     ):
@@ -295,6 +295,5 @@ def upload_artifact(
         write_adata_zarr(artifact._memory_rep, storage_path, callback=print_progress)
     elif hasattr(artifact, "_to_store") and artifact._to_store:
         logger.save(msg)
-        store_artifact(artifact._local_filepath, storage_path)
-
+        store_file_or_folder(artifact._local_filepath, storage_path)
     return storage_path

lamindb/core/_data.py

@@ -94,6 +94,23 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
         bulk_create(links, ignore_conflicts=True)
 
 
+def format_repr(value: Registry, exclude: list[str] | str | None = None) -> str:
+    if isinstance(exclude, str):
+        exclude = [exclude]
+    exclude_fields = set() if exclude is None else set(exclude)
+    exclude_fields.update(["created_at", "updated_at"])
+
+    fields = [
+        f
+        for f in value.__repr__(include_foreign_keys=False).split(", ")
+        if not any(f"{excluded_field}=" in f for excluded_field in exclude_fields)
+    ]
+    repr = ", ".join(fields)
+    if not repr.endswith(")"):
+        repr += ")"
+    return repr
+
+
 @doc_args(Data.describe.__doc__)
 def describe(self: Data):
     """{}."""
@@ -117,7 +134,7 @@ def describe(self: Data):
         msg += f"{colors.green('Provenance')}:\n  "
         related_msg = "".join(
             [
-                f"📎 {field}: {self.__getattribute__(field)}\n  "
+                f"📎 {field}: {format_repr(self.__getattribute__(field))}\n  "
                 for field in foreign_key_fields
                 if self.__getattribute__(field) is not None
             ]

lamindb/core/_feature_manager.py

@@ -91,6 +91,8 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
 def print_features(self: Data) -> str:
     from lamindb._from_values import _print_values
 
+    from ._data import format_repr
+
     msg = ""
     features_lookup = Feature.objects.using(self._state.db).lookup().dict()
     for slot, feature_set in self.features._feature_set_by_slot.items():
@@ -98,12 +100,16 @@ def print_features(self: Data) -> str:
             features = feature_set.members
             name_field = get_default_str_field(features[0])
             feature_names = [getattr(feature, name_field) for feature in features]
-            msg += f"  {colors.bold(slot)}: {feature_set}\n"
+            msg += (
+                f"  {colors.bold(slot)}: {format_repr(feature_set, exclude='hash')}\n"
+            )
             print_values = _print_values(feature_names, n=20)
             msg += f"    {print_values}\n"
         else:
             df_slot = feature_set.features.df()
-            msg += f"  {colors.bold(slot)}: {feature_set}\n"
+            msg += (
+                f"  {colors.bold(slot)}: {format_repr(feature_set, exclude='hash')}\n"
+            )
             for _, row in df_slot.iterrows():
                 if row["type"] == "category" and row["registries"] is not None:
                     labels = self.labels.get(

lamindb/core/_run_context.py

@@ -360,7 +360,6 @@ class run_context:
                 run=cls.run,
                 transform=cls.transform,
                 filepath=cls.path,
-                is_consecutive=True,
             )
         return None
 

lamindb/core/_settings.py

@@ -64,7 +64,7 @@ class Settings:
     FAQ: :doc:`/faq/idempotency`
     """
     track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.stage()` and `.backed()`.
+    """Track files as input upon `.load()`, `.cache()` and `.backed()`.
 
     Requires a global run context with :func:`~lamindb.track` was created!
 

lamindb/core/datasets/_core.py

@@ -299,7 +299,7 @@ def anndata_human_immune_cells(
         adata = sc.read('Global.h5ad')
         adata.obs = adata.obs[['donor_id', 'tissue', 'cell_type', 'assay', 'tissue_ontology_term_id', 'cell_type_ontology_term_id', 'assay_ontology_term_id']].copy()
         sc.pp.subsample(adata, fraction=0.005)
-        del adata.uns["development_stage_ontology_term_id_colors"]
+        del adata.uns["development_cache_ontology_term_id_colors"]
         del adata.uns["sex_ontology_term_id_colors"]
         adata.write('human_immune.h5ad')
     """

lamindb/core/storage/__init__.py

@@ -10,5 +10,5 @@ from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
 from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor
-from .file import delete_storage, load_to_memory
-from .object import infer_suffix, write_to_file
+from .objects import infer_suffix, write_to_file
+from .paths import delete_storage, load_to_memory

lamindb/core/storage/_backed_access.py

@@ -22,7 +22,7 @@ from lamindb_setup.core.upath import UPath, create_mapper, infer_filesystem
 from lnschema_core import Artifact
 from packaging import version
 
-from lamindb.core.storage.file import filepath_from_artifact
+from lamindb.core.storage.paths import filepath_from_artifact
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -743,15 +743,15 @@ def backed_access(
 
     if filepath.suffix in (".h5", ".hdf5", ".h5ad"):
         conn, storage = registry.open("h5py", filepath)
-    elif filepath.suffix in (".zarr", ".zrad"):
+    elif filepath.suffix == ".zarr":
         conn, storage = registry.open("zarr", filepath)
     else:
         raise ValueError(
-            "file should have .h5, .hdf5, .h5ad, .zarr or .zrad suffix, not"
+            "file should have .h5, .hdf5, .h5ad, .zarr suffix, not"
             f" {filepath.suffix}."
         )
 
-    if filepath.suffix in (".h5ad", ".zrad"):
+    if filepath.suffix in (".h5ad", ".zarr"):
         return AnnDataAccessor(conn, storage, name)
     else:
         if get_spec(storage).encoding_type == "anndata":

lamindb/core/storage/{object.py → objects.py}

@@ -21,11 +21,10 @@ def infer_suffix(dmem, adata_format: str | None = None):
     """Infer LaminDB storage file suffix from a data object."""
     if isinstance(dmem, AnnData):
         if adata_format is not None:
-            # below should be zrad, not zarr
-            if adata_format not in ("h5ad", "zarr", "zrad"):
+            if adata_format not in ("h5ad", "zarr"):
                 raise ValueError(
                     "Error when specifying AnnData storage format, it should be"
-                    f" 'h5ad', 'zarr' or 'zrad', not '{adata_format}'. Check 'format'"
+                    f" 'h5ad', 'zarr', not '{adata_format}'. Check 'format'"
                     " or the suffix of 'key'."
                 )
             return "." + adata_format

lamindb/core/storage/{file.py → paths.py}

@@ -109,23 +109,23 @@ def read_adata_h5ad(filepath, **kwargs) -> ad.AnnData:
         return adata
 
 
-def store_artifact(localpath: UPathStr, storagepath: UPath) -> None:
-    """Store directory or file to configured storage location."""
-    localpath = Path(localpath)
-    if not isinstance(storagepath, LocalPathClasses):
+def store_file_or_folder(local_path: UPathStr, storage_path: UPath) -> None:
+    """Store file or folder (localpath) at storagepath."""
+    local_path = Path(local_path)
+    if not isinstance(storage_path, LocalPathClasses):
         # this uploads files and directories
-        storagepath.upload_from(localpath, recursive=True, print_progress=True)
+        storage_path.upload_from(local_path, recursive=True, print_progress=True)
     else:  # storage path is local
-        storagepath.parent.mkdir(parents=True, exist_ok=True)
-        if localpath.is_file():
+        storage_path.parent.mkdir(parents=True, exist_ok=True)
+        if local_path.is_file():
             try:
-                shutil.copyfile(localpath, storagepath)
+                shutil.copyfile(local_path, storage_path)
             except shutil.SameFileError:
                 pass
         else:
-            if storagepath.exists():
-                shutil.rmtree(storagepath)
-            shutil.copytree(localpath, storagepath)
+            if storage_path.exists():
+                shutil.rmtree(storage_path)
+            shutil.copytree(local_path, storage_path)
 
 
 def delete_storage_using_key(
@@ -212,7 +212,7 @@ def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
     """
     filepath = create_path(filepath)
 
-    if filepath.suffix not in {".h5ad", ".zarr", ".zrad"}:
+    if filepath.suffix not in {".h5ad", ".zarr"}:
         stream = False
 
     if not stream:
@@ -229,7 +229,6 @@ def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
         ".parquet": pd.read_parquet,
         ".fcs": read_fcs,
         ".zarr": read_adata_zarr,
-        ".zrad": read_adata_zarr,
         ".html": load_html,
         ".json": load_json,
         ".h5mu": read_mdata_h5mu,

lamindb/integrations/_vitessce.py

@@ -2,39 +2,48 @@ from __future__ import annotations
 
 import json
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 
 from lamindb._artifact import Artifact
 
+if TYPE_CHECKING:
+    from vitessce import VitessceConfig
+
 
-# tested in lamin-spatial
-# can't type vitessce_config because can't assume it's installed
-def save_vitessce_config(vitessce_config, description: str) -> Artifact:
+# tested & context in https://github.com/laminlabs/lamin-spatial
+def save_vitessce_config(vitessce_config: VitessceConfig, description: str) -> Artifact:
     """Takes a ``VitessceConfig`` object and saves it as an artifact.
 
     Args:
         vitessce_config (``VitessceConfig``): A VitessceConfig object.
         description: A description for the artifact.
     """
+    # can't assume vitessce is installed
     from vitessce import VitessceConfig
 
-    assert isinstance(vitessce_config, VitessceConfig)
+    # create a local _data export_ in a folder
     timestamp = datetime.now(timezone.utc).isoformat().split(".")[0]
-    vitesse_export = f"./vitessce_export_{timestamp}.vitessce"
-    vitessce_config.export(to="files", base_url="", out_dir=vitesse_export)
-    logger.important(f"local export: {vitesse_export}")
-    artifact = Artifact(vitesse_export, description=description)
+    vitesse_export_folder = f"./vitessce_export_{timestamp}.vitessce"
+    vitessce_config.export(to="files", base_url="", out_dir=vitesse_export_folder)
+    logger.important(f"local export: {vitesse_export_folder}")
+    # create an artifact and store the local export in th cloud
+    artifact = Artifact(vitesse_export_folder, description=description)
     artifact.save()
+    # create a JSON export that points to the data in the cloud
     config_dict = vitessce_config.to_dict(base_url=artifact.path.to_url())
     logger.important(f"base url: {artifact.path.to_url()}")
+    # manually place that JSON export into the local data export folder
     config_filename = "vitessce_config.json"
-    config_file_local_path = f"{vitesse_export}/{config_filename}"
+    config_file_local_path = f"{vitesse_export_folder}/{config_filename}"
     with open(config_file_local_path, "w") as file:
         json.dump(config_dict, file)
+    # manually place that JSON export into the previously registered artifact folder
     config_file_path = artifact.path / config_filename
     config_file_path.upload_from(config_file_local_path)
+    # log the the URLs
     logger.important(f"config url: {config_file_path.to_url()}")
     slug = ln_setup.settings.instance.slug
     logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")

{lamindb-0.69.10.dist-info → lamindb-0.70.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.10
+Version: 0.70.1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.64.11
-Requires-Dist: lamindb_setup==0.69.0
+Requires-Dist: lnschema_core==0.65.1
+Requires-Dist: lamindb_setup==0.69.2
 Requires-Dist: lamin_utils==0.13.1
-Requires-Dist: lamin_cli==0.12.2
+Requires-Dist: lamin_cli==0.12.3
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -27,7 +27,7 @@ Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
 Requires-Dist: s3fs==2023.12.2 ; extra == "aws"
 Requires-Dist: fsspec[s3]==2023.12.2 ; extra == "aws"
-Requires-Dist: bionty==0.42.8 ; extra == "bionty"
+Requires-Dist: bionty==0.42.9 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-0.69.10.dist-info → lamindb-0.70.1.dist-info}/RECORD

@@ -1,32 +1,32 @@
-lamindb/__init__.py,sha256=CV6kzbhp0QdBkY2VOTJYYsswxKzqvzSgqoDE3Bf-cFE,2164
+lamindb/__init__.py,sha256=z8Z1Bsb9A_xPvukQCyt7I2d5w_FsWd27bMZlULQJ7JU,2163
 lamindb/_annotate.py,sha256=BXYWFATifbfRVmLFIgE4cZ4Fls4lkH2WXAEL2o7v-XM,43035
-lamindb/_artifact.py,sha256=OiM6BMz7Vm852QGvyr24QZlXU0dHRID6CF50U0cBVnE,36491
-lamindb/_can_validate.py,sha256=qA6Ni9scObcIeYzuudVymgOjQVLLxnGNofBSDNDaqfY,14557
-lamindb/_collection.py,sha256=xzya4M_vU8lp0nksqxj2BfSWOTwLTr9Sp6Kwp47so5g,13985
+lamindb/_artifact.py,sha256=BUl_3WYwrZ28P93Pb9AiwriVFBSLvBUEQjYEyYCrkZ0,37307
+lamindb/_can_validate.py,sha256=nvoZG-35n3HofkY4Xc6hBv9AV54_RDan7Hzp5TuqY9I,14709
+lamindb/_collection.py,sha256=wf7ClfiD3vsetts_iSUk4UihWsHq0IdOF8LdIIHS7JU,14536
 lamindb/_feature.py,sha256=srAKchY7gqD-h-cWlEiAWuHlpFKFwv0PWIA-JX0Go8c,6758
 lamindb/_feature_set.py,sha256=AzjOcHzQajpeikPOAic-aj0z_C5b7VpHVegg3ThRSLw,9045
 lamindb/_filter.py,sha256=xnjJzjF3Zj4dK_Kfymvhgczk27MhhXz5ZYc7XINbgHY,1331
-lamindb/_finish.py,sha256=4mCoDw24gXeOS_EALuUYIA57OqmcZWIi0Fk5fEDwmCg,8798
-lamindb/_from_values.py,sha256=OboM5lTysxjyu-wMWtD_Cw7SWJk8fG36tAQwb6FKM8k,12160
+lamindb/_finish.py,sha256=SIPIIMAXM2d00L6VHMf2qFiOHuTyAlLy-5qRJ-BYaIQ,8107
+lamindb/_from_values.py,sha256=DVXjnQ2wwNw-2bFzy0uXLdVlqoprrn95hTnrXwn-KqM,12638
 lamindb/_is_versioned.py,sha256=0PgRCmxEmYDcAjllLSOYZm132B1lW6QgmBBERhRyFt0,1341
 lamindb/_parents.py,sha256=N9T8jbd3eaoHDLE9TD1y1QgGcO81E6Brapy8LILzRCQ,14790
 lamindb/_query_manager.py,sha256=3zokXqxgj9vTJBnN2sbYKS-q69fyDDPF_aGq_rFHzXU,4066
 lamindb/_query_set.py,sha256=fy6xMK9MPGbD8D_i5iNzR8XA009W05ud4tbgrzd5-Vg,11287
-lamindb/_registry.py,sha256=CL-KlCC23HPWQgSeYg7Od2kbZdNViAjFqqdiGpKJKw8,19293
+lamindb/_registry.py,sha256=-Bv10zSr6IY7QM5pu_35NiVjQDJnBcXRECVe9h7GEuY,19336
 lamindb/_run.py,sha256=b7A52M1On3QzFgIYyfQoz5Kk7V3wcu9p_Prq5bzd8v8,1838
-lamindb/_save.py,sha256=aqvE0ryZs4-sDk6DZPn-Ki724gHeLyA9w-1oN5m_XMU,11425
+lamindb/_save.py,sha256=x16FBwltaTd1tnXm_zCxkvuVxyon6vRtekf37CfepXg,11426
 lamindb/_storage.py,sha256=VW8xq3VRv58-ciholvOdlcgvp_OIlLxx5GxLt-e2Irs,614
 lamindb/_transform.py,sha256=rxojJ91qQSkeYDHYbwqjFAYxBMgJd3cq_K7Z0n5g8Aw,3482
 lamindb/_ulabel.py,sha256=e5dw9h1tR0_u-DMn7Gzx0WhUhV5w7j4v3QbnLWQV7eI,1941
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=GV1FrqIMmdooEkA-5zvcTWgV1nqx1sehi6WdWEaFpxM,2171
 lamindb/core/__init__.py,sha256=MB1gEMKUf0GBQrI3dH8WRZOZQmWR4HIojXK_hXXVdqA,1235
-lamindb/core/_data.py,sha256=zxsrtf8a3PBzni1KLNVST45dxUgK11K7U6o9pmfK-xU,16905
-lamindb/core/_feature_manager.py,sha256=MHEU3I9m9p69TaTz1WTzAsimUan51RhxWV8G--_0BA8,15615
+lamindb/core/_data.py,sha256=En3v29eiJARy5l7nSsttAsDsqDLTZ4-xM8fCNyVzExI,17465
+lamindb/core/_feature_manager.py,sha256=LlYgU71AoTnrseWFCq-oZkUAYWITtRR7BNFm0AhHe-c,15773
 lamindb/core/_label_manager.py,sha256=0RtegYnK3zIisOnd970EobOrHMpp7OCH-mEoPrPXw2c,9075
 lamindb/core/_mapped_collection.py,sha256=_OwFZh5SePDUD70XIK5kngv3we_Z5-YdGHNfpUSatSQ,19469
-lamindb/core/_run_context.py,sha256=GS593lxHsbQyhbrp6ppzZ0r-fVe6W099NDXM6xyg8-U,17509
-lamindb/core/_settings.py,sha256=32109tBsMcwdGlNTWAGR7YBuCVXrrVfzIEKHWVk1bBQ,5727
+lamindb/core/_run_context.py,sha256=xcsPhpabm2a-PnwH-vDJ4Mx4qQLdFsn4ZUpWMWKZgoM,17472
+lamindb/core/_settings.py,sha256=r9si7wJb31tI4vfz9dUN4iXe6QQU7FjnqAEsHy2UDzM,5727
 lamindb/core/_sync_git.py,sha256=IlTqw55inPp_RZbN_YScaCeKza7LeF9mClQw55W3_d4,3921
 lamindb/core/_track_environment.py,sha256=xLZ6kgzxWS6MWZ5LQ_wkbJX99vmYOT8iQ-Fz4OHCgWw,754
 lamindb/core/_transform_settings.py,sha256=eV96QKX9jOojjzF-a0oo0wXQsMXN2F6QV7orE06oFC8,161
@@ -36,19 +36,19 @@ lamindb/core/fields.py,sha256=Jgi_XI-iTe6cT7oD8FV_JqEpjN1Q9rZWwL8VLtj4jkA,164
 lamindb/core/types.py,sha256=xeQF2x40p2pR9eIVQrXT74RrS810z2fbjmTRTSQUqPM,230
 lamindb/core/versioning.py,sha256=DsEHpCueNwhRiIaRH5-O8H_1fJVNtWslCRx30YiIS5o,3080
 lamindb/core/datasets/__init__.py,sha256=zRP98oqUAaXhqWyKMiH0s_ImVIuNeziQQ2kQ_t0f-DI,1353
-lamindb/core/datasets/_core.py,sha256=ZPsDETZp-SqQelK5O0l7sDZFjuz8Sy93UbTfSYqP9cc,20191
+lamindb/core/datasets/_core.py,sha256=36vUOYFkX_4hBAnM_BujV5BRARMI5b9iI_SM9qS7wGc,20191
 lamindb/core/datasets/_fake.py,sha256=BZF9R_1iF0HDnvtZNqL2FtsjSMuqDIfuFxnw_LJYIh4,953
-lamindb/core/storage/__init__.py,sha256=9alBNtyH59VnoWJS-IdjLwFKlK-kgeCGl6jXk0_wGeQ,369
+lamindb/core/storage/__init__.py,sha256=zsVbIseNLJfObDl7cuCbULr6rnux_R_N8EVV843XmIU,371
 lamindb/core/storage/_anndata_sizes.py,sha256=aXO3OB--tF5MChenSsigW6Q-RuE8YJJOUTVukkLrv9A,1029
-lamindb/core/storage/_backed_access.py,sha256=uW0zaeKune6k2zYJcLazIA7Xux9TS2tAHDNdz1syYZw,24557
+lamindb/core/storage/_backed_access.py,sha256=DweJdYP8tKEOZbfJ1UKFQgAIzQjsJqdqj8U9y7T0oXo,24538
 lamindb/core/storage/_zarr.py,sha256=0i9-cJPjieIsp5UpK-IyRPkHAF-iKkWgpkWviSni2MM,2900
-lamindb/core/storage/file.py,sha256=AEMbrC6oWoIp3vs4F2bgm50EVBxkCGbN9R4na7Wanaw,7754
-lamindb/core/storage/object.py,sha256=37p8CSlfSlWPVuuD3MFcwQRj1n_kovLKVImh9SCR4Eg,1601
+lamindb/core/storage/objects.py,sha256=5LbBeZVKuOOB8DceSE-PN8elKY0N9OhFXZPQJE4lK48,1538
+lamindb/core/storage/paths.py,sha256=XWfSHK5b3_TFiK-IMvH-srvxO0bZStzA_rwjNaTxQU4,7725
 lamindb/integrations/__init__.py,sha256=aH2PmO2m4-vwIifMYTB0Fyyr_gZWtVnV71jT0tVWSw0,123
-lamindb/integrations/_vitessce.py,sha256=xLlFr_GzoY5RjyouPFEO-W7crBlTVQdHR2ns1IcCZFc,1673
+lamindb/integrations/_vitessce.py,sha256=l3GPkzQXcwnMypbm8vDkITjognELjX8ucLaiqy99Jgg,2131
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.69.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.69.10.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.69.10.dist-info/METADATA,sha256=R4DYvAOjBwW0ywfjHvYo4xLUABytQWm6iGg8LE8yfbU,2837
-lamindb-0.69.10.dist-info/RECORD,,
+lamindb-0.70.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.70.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.70.1.dist-info/METADATA,sha256=qbHYrCM8tdhGUBT-70lupGW7AQ5RVNA7gg9yRTvihwA,2835
+lamindb-0.70.1.dist-info/RECORD,,