lamindb 0.76.0__py3-none-any.whl → 0.76.2__py3-none-any.whl

lamindb/__init__.py

@@ -1,6 +1,7 @@
 """A data framework for biology.
 
-Registries:
+Records
+=======
 
 .. autosummary::
    :toctree: .
@@ -16,19 +17,20 @@ Registries:
    FeatureSet
    Param
 
-Key functionality:
+Key functionality
+=================
 
 .. autosummary::
    :toctree: .
 
+   context
    connect
-   track
-   finish
    Curate
    view
    save
 
-Modules & settings:
+Modules & settings
+==================
 
 .. autosummary::
    :toctree: .
@@ -42,7 +44,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.0"
+__version__ = "0.76.2"
 
 import os as _os
 
@@ -93,10 +95,9 @@ if _check_instance_setup(from_lamindb=True):
         integrations,
     )
     from ._curate import Curate
-    from ._finish import finish
     from ._save import save
     from ._view import view
-    from .core._run_context import run_context as _run_context
+    from .core._context import context
     from .core._settings import settings
 
     # schema modules
@@ -107,8 +108,8 @@ if _check_instance_setup(from_lamindb=True):
 
         _reload_schema_modules(_lamindb_setup.settings.instance)
 
-    track = _run_context._track
+    track = context.track  # backward compat
+    finish = context.finish  # backward compat
     settings.__doc__ = """Global :class:`~lamindb.core.Settings`."""
+    context.__doc__ = """Global :class:`~lamindb.core.Context`."""
     from django.db.models import Q
-
-    Annotate = Curate  # backward compat

lamindb/_artifact.py

@@ -9,6 +9,7 @@ import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
 from anndata import AnnData
+from django.db.models import Q, QuerySet
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup._init_instance import register_storage_in_instance
@@ -44,7 +45,10 @@ from lamindb.core.storage.paths import (
     check_path_is_child_of_root,
     filepath_from_artifact,
 )
-from lamindb.core.versioning import get_uid_from_old_version, init_uid
+from lamindb.core.versioning import (
+    create_uid,
+    message_update_key_in_version_family,
+)
 
 from .core._data import (
     add_transform_to_kwargs,
@@ -107,7 +111,7 @@ def process_pathlike(
                 new_root = list(filepath.parents)[-1]
                 # do not register remote storage locations on hub if the current instance
                 # is not managed on the hub
-                storage_settings = init_storage(
+                storage_settings, _ = init_storage(
                     new_root, prevent_register_hub=not setup_settings.instance.is_on_hub
                 )
                 storage_record = register_storage_in_instance(storage_settings)
@@ -192,12 +196,14 @@ def process_data(
 
 def get_stat_or_artifact(
     path: UPath,
+    key: str | None = None,
     check_hash: bool = True,
-    using_key: str | None = None,
-) -> tuple[int, str | None, str | None, int | None] | Artifact:
+    is_replace: bool = False,
+    instance: str | None = None,
+) -> tuple[int, str | None, str | None, int | None, Artifact | None] | Artifact:
     n_objects = None
     if settings.creation.artifact_skip_size_hash:
-        return None, None, None, n_objects
+        return None, None, None, n_objects, None
     stat = path.stat()  # one network request
     if not isinstance(path, LocalPathClasses):
         size, hash, hash_type = None, None, None
@@ -210,7 +216,7 @@ def get_stat_or_artifact(
                 size, hash, hash_type, n_objects = get_stat_dir_cloud(path)
         if hash is None:
             logger.warning(f"did not add hash for {path}")
-            return size, hash, hash_type, n_objects
+            return size, hash, hash_type, n_objects, None
     else:
         if path.is_dir():
             size, hash, hash_type, n_objects = hash_dir(path)
@@ -218,17 +224,26 @@ def get_stat_or_artifact(
             hash, hash_type = hash_file(path)
             size = stat.st_size
     if not check_hash:
-        return size, hash, hash_type, n_objects
-    # also checks hidden and trashed files
-    # in Alex's mind the following two lines should be equivalent
-    # but they aren't according to pytest tests/test_artifact.py::test_from_dir_single_artifact
-    if using_key is None:
-        result = Artifact.filter(hash=hash, visibility=None).all()
+        return size, hash, hash_type, n_objects, None
+    previous_artifact_version = None
+    if key is None or is_replace:
+        result = Artifact.objects.using(instance).filter(hash=hash).all()
+        artifact_with_same_hash_exists = len(result) > 0
     else:
+        storage_id = settings.storage.id
         result = (
-            Artifact.objects.using(using_key).filter(hash=hash, visibility=None).all()
+            Artifact.objects.using(instance)
+            .filter(Q(hash=hash) | Q(key=key, storage_id=storage_id))
+            .order_by("-created_at")
+            .all()
         )
-    if len(result) > 0:
+        artifact_with_same_hash_exists = len(result.filter(hash=hash).all()) > 0
+        if not artifact_with_same_hash_exists and len(result) > 0:
+            logger.important(
+                f"creating new artifact version for key='{key}' (storage: '{settings.storage.root_as_str}')"
+            )
+            previous_artifact_version = result[0]
+    if artifact_with_same_hash_exists:
         if settings.creation.artifact_if_hash_exists == "error":
             msg = f"artifact with same hash exists: {result[0]}"
             hint = (
@@ -241,7 +256,7 @@ def get_stat_or_artifact(
                 "creating new Artifact object despite existing artifact with same hash:"
                 f" {result[0]}"
             )
-            return size, hash, hash_type, n_objects
+            return size, hash, hash_type, n_objects, None
         else:
             if result[0].visibility == -1:
                 raise FileExistsError(
@@ -251,11 +266,11 @@ def get_stat_or_artifact(
             logger.important(f"returning existing artifact with same hash: {result[0]}")
             return result[0]
     else:
-        return size, hash, hash_type, n_objects
+        return size, hash, hash_type, n_objects, previous_artifact_version
 
 
 def check_path_in_existing_storage(
-    path: Path | UPath, using_key: str | None
+    path: Path | UPath, using_key: str | None = None
 ) -> Storage | bool:
     for storage in Storage.objects.using(using_key).filter().all():
         # if path is part of storage, return it
@@ -290,8 +305,10 @@ def get_artifact_kwargs_from_data(
     run: Run | None,
     format: str | None,
     provisional_uid: str,
+    version: str | None,
     default_storage: Storage,
     using_key: str | None = None,
+    is_replace: bool = False,
     skip_check_exists: bool = False,
 ):
     run = get_run(run)
@@ -306,7 +323,9 @@ def get_artifact_kwargs_from_data(
     )
     stat_or_artifact = get_stat_or_artifact(
         path=path,
-        using_key=using_key,
+        key=key,
+        instance=using_key,
+        is_replace=is_replace,
     )
     if isinstance(stat_or_artifact, Artifact):
         artifact = stat_or_artifact
@@ -321,7 +340,12 @@ def get_artifact_kwargs_from_data(
             stat_or_artifact.transform = run.transform
         return artifact, None
     else:
-        size, hash, hash_type, n_objects = stat_or_artifact
+        size, hash, hash_type, n_objects, revises = stat_or_artifact
+
+    if revises is not None:  # update provisional_uid
+        provisional_uid, revises = create_uid(revises=revises, version=version)
+        if path.as_posix().startswith(settings._storage_settings.cache_dir.as_posix()):
+            path = path.rename(f"{provisional_uid}{suffix}")
 
     check_path_in_storage = False
     if use_existing_storage_key:
@@ -365,6 +389,7 @@ def get_artifact_kwargs_from_data(
         key_is_virtual = False
 
     kwargs = {
+        "uid": provisional_uid,
         "suffix": suffix,
         "hash": hash,
         "_hash_type": hash_type,
@@ -509,9 +534,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
     description: str | None = (
         kwargs.pop("description") if "description" in kwargs else None
     )
-    is_new_version_of: Artifact | None = (
-        kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
-    )
+    revises: Artifact | None = kwargs.pop("revises") if "revises" in kwargs else None
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     visibility: int | None = (
         kwargs.pop("visibility")
@@ -534,28 +557,38 @@ def __init__(artifact: Artifact, *args, **kwargs):
     )
     accessor = kwargs.pop("_accessor") if "_accessor" in kwargs else None
     accessor = _check_accessor_artifact(data=data, accessor=accessor)
+    if "is_new_version_of" in kwargs:
+        logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
+        revises = kwargs.pop("is_new_version_of")
     if not len(kwargs) == 0:
         raise ValueError(
-            "Only data, key, run, description, version, is_new_version_of, visibility"
+            "Only data, key, run, description, version, revises, visibility"
             f" can be passed, you passed: {kwargs}"
         )
-
-    if is_new_version_of is None:
-        provisional_uid = init_uid(version=version, n_full_id=20)
-    else:
-        if not isinstance(is_new_version_of, Artifact):
-            raise TypeError("is_new_version_of has to be of type ln.Artifact")
-        provisional_uid, version = get_uid_from_old_version(
-            is_new_version_of, version, using_key
+    if revises is not None and key is not None and revises.key != key:
+        note = message_update_key_in_version_family(
+            suid=revises.stem_uid,
+            existing_key=revises.key,
+            new_key=key,
+            registry="Artifact",
         )
+        raise ValueError(
+            f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
+        )
+
+    provisional_uid, revises = create_uid(revises=revises, version=version)
+    if revises is not None:
+        if not isinstance(revises, Artifact):
+            raise TypeError("`revises` has to be of type `Artifact`")
         if description is None:
-            description = is_new_version_of.description
+            description = revises.description
     kwargs_or_artifact, privates = get_artifact_kwargs_from_data(
         data=data,
         key=key,
         run=run,
         format=format,
         provisional_uid=provisional_uid,
+        version=version,
         default_storage=default_storage,
         using_key=using_key,
         skip_check_exists=skip_check_exists,
@@ -576,25 +609,23 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    # only set key now so that we don't do a look-up on it in case revises is passed
+    if revises is not None:
+        kwargs["key"] = revises.key
     # in case we have a new version of a folder with a different hash, print a
     # warning that the old version can't be recovered
-    if (
-        is_new_version_of is not None
-        and is_new_version_of.n_objects is not None
-        and is_new_version_of.n_objects > 1
-    ):
+    if revises is not None and revises.n_objects is not None and revises.n_objects > 1:
         logger.warning(
-            f"artifact version {version} will _update_ the state of folder {is_new_version_of.path} - "
-            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `is_new_version_of`"
+            f"artifact version {version} will _update_ the state of folder {revises.path} - "
+            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `revises`"
         )
 
     kwargs["type"] = type
-    kwargs["uid"] = provisional_uid
     kwargs["version"] = version
     kwargs["description"] = description
     kwargs["visibility"] = visibility
     kwargs["_accessor"] = accessor
-    kwargs["is_new_version_of"] = is_new_version_of
+    kwargs["revises"] = revises
     # this check needs to come down here because key might be populated from an
     # existing file path during get_artifact_kwargs_from_data()
     if (
@@ -623,8 +654,7 @@ def from_df(
     key: str | None = None,
     description: str | None = None,
     run: Run | None = None,
-    version: str | None = None,
-    is_new_version_of: Artifact | None = None,
+    revises: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
     """{}"""  # noqa: D415
@@ -633,8 +663,7 @@ def from_df(
         key=key,
         run=run,
         description=description,
-        version=version,
-        is_new_version_of=is_new_version_of,
+        revises=revises,
         _accessor="DataFrame",
         type="dataset",
         **kwargs,
@@ -650,8 +679,7 @@ def from_anndata(
     key: str | None = None,
     description: str | None = None,
     run: Run | None = None,
-    version: str | None = None,
-    is_new_version_of: Artifact | None = None,
+    revises: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
     """{}"""  # noqa: D415
@@ -662,8 +690,7 @@ def from_anndata(
         key=key,
         run=run,
         description=description,
-        version=version,
-        is_new_version_of=is_new_version_of,
+        revises=revises,
         _accessor="AnnData",
         type="dataset",
         **kwargs,
@@ -679,8 +706,7 @@ def from_mudata(
     key: str | None = None,
     description: str | None = None,
     run: Run | None = None,
-    version: str | None = None,
-    is_new_version_of: Artifact | None = None,
+    revises: Artifact | None = None,
     **kwargs,
 ) -> Artifact:
     """{}"""  # noqa: D415
@@ -689,8 +715,7 @@ def from_mudata(
         key=key,
         run=run,
         description=description,
-        version=version,
-        is_new_version_of=is_new_version_of,
+        revises=revises,
         _accessor="MuData",
         type="dataset",
         **kwargs,
@@ -815,6 +840,8 @@ def replace(
         run=run,
         format=format,
         default_storage=default_storage,
+        version=None,
+        is_replace=True,
     )
 
     # this artifact already exists
@@ -913,7 +940,7 @@ def open(
                     logger.warning(
                         "The hash of the tiledbsoma store has changed, creating a new version of the artifact."
                     )
-                    new_version = Artifact(filepath, is_new_version_of=self).save()
+                    new_version = Artifact(filepath, revises=self).save()
                     init_self_from_db(self, new_version)
 
                     if localpath != filepath and localpath.exists():

lamindb/_can_validate.py

@@ -332,7 +332,9 @@ def _standardize(
         # here, we can safely import bionty
         from bionty._bionty import create_or_get_organism_record
 
-        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
+        organism_record = create_or_get_organism_record(
+            organism=organism, registry=registry
+        )
         organism = (
             organism_record.name if organism_record is not None else organism_record
         )
@@ -403,7 +405,10 @@ def _standardize(
             logger.warning(warn_msg)
 
         mapper.update(std_names_bt_mapper)
-        result = pd.Series(std_names_db).replace(std_names_bt_mapper).tolist()
+        if pd.api.types.is_categorical_dtype(std_names_db):
+            result = std_names_db.cat.rename_categories(std_names_bt_mapper).tolist()
+        else:
+            result = pd.Series(std_names_db).replace(std_names_bt_mapper).tolist()
         return _return(result=result, mapper=mapper)
 
     else:
@@ -514,7 +519,9 @@ def _filter_query_based_on_organism(
         # here, we can safely import bionty
         from bionty._bionty import create_or_get_organism_record
 
-        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
+        organism_record = create_or_get_organism_record(
+            organism=organism, registry=registry
+        )
         if organism_record is not None:
             queryset = queryset.filter(organism__name=organism_record.name)
 

lamindb/_collection.py

@@ -11,7 +11,6 @@ from typing import (
 import anndata as ad
 import lamindb_setup as ln_setup
 import pandas as pd
-from anndata import AnnData
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
@@ -27,7 +26,7 @@ from lamindb._artifact import update_attributes
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input
 from lamindb.core._mapped_collection import MappedCollection
-from lamindb.core.versioning import get_uid_from_old_version, init_uid
+from lamindb.core.versioning import process_revises
 
 from . import Artifact, Run
 from ._record import init_self_from_db
@@ -37,10 +36,10 @@ from .core._data import (
     save_feature_set_links,
     save_feature_sets,
 )
+from .core._settings import settings
 
 if TYPE_CHECKING:
     from lamindb.core.storage import UPath
-    from lamindb.core.storage._backed_access import AnnDataAccessor, BackedAccessor
 
     from ._query_set import QuerySet
 
@@ -72,9 +71,7 @@ def __init__(
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
     )
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
-    is_new_version_of: Collection | None = (
-        kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
-    )
+    revises: Collection | None = kwargs.pop("revises") if "revises" in kwargs else None
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     visibility: int | None = (
         kwargs.pop("visibility")
@@ -84,18 +81,16 @@ def __init__(
     feature_sets: dict[str, FeatureSet] = (
         kwargs.pop("feature_sets") if "feature_sets" in kwargs else {}
     )
+    if "is_new_version_of" in kwargs:
+        logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
+        revises = kwargs.pop("is_new_version_of")
     if not len(kwargs) == 0:
         raise ValueError(
             f"Only artifacts, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
         )
-    if is_new_version_of is None:
-        provisional_uid = init_uid(version=version, n_full_id=20)
-    else:
-        if not isinstance(is_new_version_of, Collection):
-            raise TypeError("is_new_version_of has to be of type ln.Collection")
-        provisional_uid, version = get_uid_from_old_version(is_new_version_of, version)
-        if name is None:
-            name = is_new_version_of.name
+    provisional_uid, version, name, revises = process_revises(
+        revises, version, name, Collection
+    )
     run = get_run(run)
     if isinstance(artifacts, Artifact):
         artifacts = [artifacts]
@@ -147,6 +142,9 @@ def __init__(
     else:
         kwargs = {}
         add_transform_to_kwargs(kwargs, run)
+        search_names_setting = settings.creation.search_names
+        if revises is not None and name == revises.name:
+            settings.creation.search_names = False
         super(Collection, collection).__init__(
             uid=provisional_uid,
             name=name,
@@ -158,14 +156,15 @@ def __init__(
             run=run,
             version=version,
             visibility=visibility,
-            is_new_version_of=is_new_version_of,
+            revises=revises,
             **kwargs,
         )
+        settings.creation.search_names = search_names_setting
     collection._artifacts = artifacts
     collection._feature_sets = feature_sets
     # register provenance
-    if is_new_version_of is not None:
-        _track_run_input(is_new_version_of, run=run)
+    if revises is not None:
+        _track_run_input(revises, run=run)
     _track_run_input(artifacts, run=run)
 
 
@@ -192,7 +191,7 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
     feature_sets_union = {}
     logger.debug("union")
     for slot, feature_set_ids_slot in feature_sets_by_slots.items():
-        feature_set_1 = FeatureSet.filter(id=feature_set_ids_slot[0]).one()
+        feature_set_1 = FeatureSet.get(id=feature_set_ids_slot[0])
         related_name = feature_set_1._get_related_name()
         features_registry = getattr(FeatureSet, related_name).field.model
         start_time = logger.debug("run filter")

lamindb/_curate.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import copy
-from typing import TYPE_CHECKING, Iterable, Type
+from typing import TYPE_CHECKING, Iterable
 
 import anndata as ad
 import lamindb_setup as ln_setup
@@ -102,9 +102,12 @@ class DataFrameCurator:
     Examples:
         >>> import bionty as bt
         >>> curate = ln.Curate.from_df(
-                df,
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name}
-            )
+        ...     df,
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     }
+        ... )
     """
 
     def __init__(
@@ -247,7 +250,7 @@ class DataFrameCurator:
                 key=categorical,
                 using_key=self._using_key,
                 validated_only=validated_only,
-                sources=self._sources.get(categorical),
+                source=self._sources.get(categorical),
                 **kwargs,
             )
 
@@ -260,6 +263,9 @@ class DataFrameCurator:
     def validate(self, organism: str | None = None) -> bool:
         """Validate variables and categorical observations.
 
+        Args:
+            organism: The organism name.
+
         Returns:
             Whether the DataFrame is validated.
         """
@@ -313,11 +319,11 @@ class DataFrameCurator:
 
     def clean_up_failed_runs(self):
         """Clean up previous failed runs that don't save any outputs."""
-        from lamindb.core._run_context import run_context
+        from lamindb.core._context import context
 
-        if run_context.transform is not None:
-            Run.filter(transform=run_context.transform, output_artifacts=None).exclude(
-                uid=run_context.run.uid
+        if context.run is not None:
+            Run.filter(transform=context.run.transform, output_artifacts=None).exclude(
+                uid=context.run.uid
             ).delete()
 
 
@@ -341,11 +347,14 @@ class AnnDataCurator(DataFrameCurator):
     Examples:
         >>> import bionty as bt
         >>> curate = ln.Curate.from_anndata(
-                adata,
-                var_index=bt.Gene.ensembl_gene_id,
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name},
-                organism="human",
-            )
+        ...     adata,
+        ...     var_index=bt.Gene.ensembl_gene_id,
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     },
+        ...     organism="human",
+        ... )
     """
 
     def __init__(
@@ -548,11 +557,17 @@ class MuDataCurator:
     Examples:
         >>> import bionty as bt
         >>> curate = ln.Curate.from_mudata(
-                mdata,
-                var_index={"rna": bt.Gene.ensembl_gene_id, "adt": ln.CellMarker.name},
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name},
-                organism="human",
-            )
+        ...     mdata,
+        ...     var_index={
+        ...         "rna": bt.Gene.ensembl_gene_id,
+        ...         "adt": ln.CellMarker.name
+        ...     },
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     },
+        ...     organism="human",
+        ... )
     """
 
     def __init__(
@@ -625,6 +640,7 @@ class MuDataCurator:
             using_key=self._using_key,
             validated_only=validated_only,
             dtype="number",
+            source=self._sources.get(modality, {}).get("var_index"),
             **kwargs,
         )
 
@@ -687,6 +703,7 @@ class MuDataCurator:
             using_key=self._using_key,
             validated_only=False,
             df=self._mdata[modality].obs,
+            source=self._sources.get(modality, {}).get("columns"),
             **self._kwargs,  # type: ignore
             **kwargs,
         )
@@ -1458,6 +1475,7 @@ def update_registry_from_using_instance(
         values: A list of values to be saved as labels.
         field: The FieldAttr object representing the field for which labels are being saved.
         using_key: The name of the instance from which to transfer labels (if applicable).
+        standardize: Whether to also standardize the values.
         kwargs: Additional keyword arguments to pass to the registry model.
 
     Returns: