lamindb 0.77.2__py3-none-any.whl → 1.0rc1__py3-none-any.whl

lamindb/__init__.py

@@ -1,28 +1,37 @@
 """A data framework for biology.
 
-Core registries.
+Tracking notebooks & scripts.
+
+.. autosummary::
+   :toctree: .
+
+   track
+   finish
+
+Registries.
 
 .. autosummary::
    :toctree: .
 
    Artifact
-   Collection
    Transform
+   ULabel
    Run
    User
    Storage
-   ULabel
    Feature
    FeatureSet
    Param
+   Collection
+   Project
+   Reference
+   Person
 
 Key functionality.
 
 .. autosummary::
    :toctree: .
 
-   track
-   finish
    connect
    Curator
    view
@@ -38,70 +47,68 @@ Modules and settings.
    settings
    setup
    UPath
+   base
    core
 
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.77.2"
-
-import os as _os
+__version__ = "1.0rc1"
 
-import lamindb_setup as _lamindb_setup
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
 from lamindb_setup._connect_instance import connect
 from lamindb_setup.core.upath import UPath
 
-from . import setup
+from . import base, setup
 
 
 def __getattr__(name):
     raise _InstanceNotSetupError()
 
 
-if _check_instance_setup(from_module="lnschema_core"):
-    del _InstanceNotSetupError
-    del __getattr__  # delete so that imports work out
-    from lnschema_core.models import (
-        Artifact,
-        Collection,
-        Feature,
-        FeatureSet,
-        Param,
-        Run,
-        Storage,
-        Transform,
-        ULabel,
-        User,
-    )
-
+if _check_instance_setup(from_module="lamindb"):
+    del __getattr__  # so that imports work out
     from . import core  # isort: split
     from . import (
         _artifact,
         _can_curate,
         _collection,
-        _curate,
         _feature,
-        _feature_set,
         _is_versioned,
         _parents,
         _record,
         _run,
+        _schema,
         _storage,
         _transform,
         _ulabel,
         integrations,
     )
-    from ._curate import Curator
     from ._save import save
     from ._view import view
     from .core._context import context
     from .core._settings import settings
+    from .curators import Curator
+    from .models import (
+        Artifact,
+        Collection,
+        Feature,
+        FeatureSet,  # backward compat
+        Param,
+        Person,
+        Project,
+        Reference,
+        Run,
+        Schema,  # forward compat
+        Storage,
+        Transform,
+        ULabel,
+        User,
+    )
 
-    track = context.track  # simple access because these are so common
-    finish = context.finish  # simple access because these are so common
-    Curate = Curator  # backward compat
+    track = context.track  # simple access
+    finish = context.finish  # simple access
     settings.__doc__ = """Global settings (:class:`~lamindb.core.Settings`)."""
     context.__doc__ = """Global run context (:class:`~lamindb.core.Context`).
 

lamindb/_artifact.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import os
 import shutil
-from collections.abc import Mapping
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any
 
@@ -23,20 +22,17 @@ from lamindb_setup.core.upath import (
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core.models import Artifact, FeatureManager, ParamManager, Run, Storage
-from lnschema_core.types import (
-    VisibilityChoice,
-)
+
+from lamindb.models import Artifact, FeatureManager, ParamManager, Run, Storage
 
 from ._parents import view_lineage
 from ._utils import attach_func_to_class_method
 from .core._data import (
     _track_run_input,
-    add_transform_to_kwargs,
     describe,
     get_run,
-    save_feature_set_links,
-    save_feature_sets,
+    save_schema_links,
+    save_staged__schemas_m2m,
 )
 from .core._settings import settings
 from .core.exceptions import IntegrityError, InvalidArgument
@@ -209,9 +205,9 @@ def get_stat_or_artifact(
     is_replace: bool = False,
     instance: str | None = None,
 ) -> tuple[int, str | None, str | None, int | None, Artifact | None] | Artifact:
-    n_objects = None
+    n_files = None
     if settings.creation.artifact_skip_size_hash:
-        return None, None, None, n_objects, None
+        return None, None, None, n_files, None
     stat = path.stat()  # one network request
     if not isinstance(path, LocalPathClasses):
         size, hash, hash_type = None, None, None
@@ -221,18 +217,18 @@ def get_stat_or_artifact(
             if (store_type := stat["type"]) == "file":
                 size, hash, hash_type = get_stat_file_cloud(stat)
             elif store_type == "directory":
-                size, hash, hash_type, n_objects = get_stat_dir_cloud(path)
+                size, hash, hash_type, n_files = get_stat_dir_cloud(path)
         if hash is None:
             logger.warning(f"did not add hash for {path}")
-            return size, hash, hash_type, n_objects, None
+            return size, hash, hash_type, n_files, None
     else:
         if path.is_dir():
-            size, hash, hash_type, n_objects = hash_dir(path)
+            size, hash, hash_type, n_files = hash_dir(path)
         else:
             hash, hash_type = hash_file(path)
             size = stat.st_size
     if not check_hash:
-        return size, hash, hash_type, n_objects, None
+        return size, hash, hash_type, n_files, None
     previous_artifact_version = None
     if key is None or is_replace:
         result = Artifact.objects.using(instance).filter(hash=hash).all()
@@ -264,17 +260,19 @@ def get_stat_or_artifact(
                 "creating new Artifact object despite existing artifact with same hash:"
                 f" {result[0]}"
             )
-            return size, hash, hash_type, n_objects, None
+            return size, hash, hash_type, n_files, None
         else:
-            if result[0].visibility == -1:
+            if result[0]._branch_code == -1:
                 raise FileExistsError(
                     f"You're trying to re-create this artifact in trash: {result[0]}"
                     "Either permanently delete it with `artifact.delete(permanent=True)` or restore it with `artifact.restore()`"
                 )
-            logger.important(f"returning existing artifact with same hash: {result[0]}")
+            logger.important(
+                f"returning existing artifact with same hash: {result[0]}; if you intended to query to track this artifact as an input, use: ln.Artifact.get()"
+            )
             return result[0]
     else:
-        return size, hash, hash_type, n_objects, previous_artifact_version
+        return size, hash, hash_type, n_files, previous_artifact_version
 
 
 def check_path_in_existing_storage(
@@ -346,10 +344,9 @@ def get_artifact_kwargs_from_data(
                 artifact.run._output_artifacts_with_later_updates.add(artifact)
             # update the run of the artifact with the latest run
             stat_or_artifact.run = run
-            stat_or_artifact.transform = run.transform
         return artifact, None
     else:
-        size, hash, hash_type, n_objects, revises = stat_or_artifact
+        size, hash, hash_type, n_files, revises = stat_or_artifact
 
     if revises is not None:  # update provisional_uid
         provisional_uid, revises = create_uid(revises=revises, version=version)
@@ -381,7 +378,7 @@ def get_artifact_kwargs_from_data(
         key=key,
         uid=provisional_uid,
         suffix=suffix,
-        is_dir=n_objects is not None,
+        is_dir=n_files is not None,
     )
 
     # do we use a virtual or an actual storage key?
@@ -403,7 +400,8 @@ def get_artifact_kwargs_from_data(
         # passing both the id and the object
         # to make them both available immediately
         # after object creation
-        "n_objects": n_objects,
+        "n_files": n_files,
+        "_overwrite_versions": n_files is not None,  # True for folder, False for file
         "n_observations": None,  # to implement
         "run_id": run.id if run is not None else None,
         "run": run,
@@ -470,7 +468,7 @@ def data_is_anndata(data: AnnData | UPathStr) -> bool:
             if fsspec.utils.get_protocol(data_path.as_posix()) == "file":
                 return zarr_is_adata(data_path)
             else:
-                logger.warning("We do not check if cloud zarr is AnnData or not.")
+                logger.warning("We do not check if cloud zarr is AnnData or not")
                 return False
     return False
 
@@ -486,25 +484,25 @@ def data_is_mudata(data: MuData | UPathStr) -> bool:
     return False
 
 
-def _check_accessor_artifact(data: Any, accessor: str | None = None):
-    if accessor is None:
+def _check_otype_artifact(data: Any, otype: str | None = None):
+    if otype is None:
         if isinstance(data, pd.DataFrame):
             logger.warning("data is a DataFrame, please use .from_df()")
-            accessor = "DataFrame"
-            return accessor
+            otype = "DataFrame"
+            return otype
 
         data_is_path = isinstance(data, (str, Path))
         if data_is_anndata(data):
             if not data_is_path:
                 logger.warning("data is an AnnData, please use .from_anndata()")
-            accessor = "AnnData"
+            otype = "AnnData"
         elif data_is_mudata(data):
             if not data_is_path:
                 logger.warning("data is a MuData, please use .from_mudata()")
-            accessor = "MuData"
+            otype = "MuData"
         elif not data_is_path:  # UPath is a subclass of Path
             raise TypeError("data has to be a string, Path, UPath")
-    return accessor
+    return otype
 
 
 def __init__(artifact: Artifact, *args, **kwargs):
@@ -526,7 +524,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed: data")
 
     data: str | Path = kwargs.pop("data") if len(args) == 0 else args[0]
-    type: str = kwargs.pop("type") if "type" in kwargs else None
+    kind: str = kwargs.pop("kind") if "kind" in kwargs else None
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     description: str | None = (
@@ -534,11 +532,12 @@ def __init__(artifact: Artifact, *args, **kwargs):
     )
     revises: Artifact | None = kwargs.pop("revises") if "revises" in kwargs else None
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
-    visibility: int | None = (
-        kwargs.pop("visibility")
-        if "visibility" in kwargs
-        else VisibilityChoice.default.value
-    )
+    if "visibility" in kwargs:
+        _branch_code = kwargs.pop("visibility")
+    elif "_branch_code" in kwargs:
+        _branch_code = kwargs.pop("_branch_code")
+    else:
+        _branch_code = 1
     format = kwargs.pop("format") if "format" in kwargs else None
     _is_internal_call = kwargs.pop("_is_internal_call", False)
     skip_check_exists = (
@@ -554,14 +553,14 @@ def __init__(artifact: Artifact, *args, **kwargs):
     using_key = (
         kwargs.pop("using_key") if "using_key" in kwargs else settings._using_key
     )
-    accessor = kwargs.pop("_accessor") if "_accessor" in kwargs else None
-    accessor = _check_accessor_artifact(data=data, accessor=accessor)
-    if "is_new_version_of" in kwargs:
-        logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
-        revises = kwargs.pop("is_new_version_of")
+    otype = kwargs.pop("otype") if "otype" in kwargs else None
+    otype = _check_otype_artifact(data=data, otype=otype)
+    if "type" in kwargs:
+        logger.warning("`type` will be removed soon, please use `kind`")
+        kind = kwargs.pop("type")
     if not len(kwargs) == 0:
         raise ValueError(
-            "Only data, key, run, description, version, revises, visibility"
+            "Only data, key, run, description, version, revises"
             f" can be passed, you passed: {kwargs}"
         )
     if revises is not None and key is not None and revises.key != key:
@@ -654,11 +653,11 @@ def __init__(artifact: Artifact, *args, **kwargs):
     if revises is not None:
         kwargs["key"] = revises.key
 
-    kwargs["type"] = type
+    kwargs["kind"] = kind
     kwargs["version"] = version
     kwargs["description"] = description
-    kwargs["visibility"] = visibility
-    kwargs["_accessor"] = accessor
+    kwargs["_branch_code"] = _branch_code
+    kwargs["otype"] = otype
     kwargs["revises"] = revises
     # this check needs to come down here because key might be populated from an
     # existing file path during get_artifact_kwargs_from_data()
@@ -669,8 +668,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
     ):
         raise ValueError("Pass one of key, run or description as a parameter")
 
-    add_transform_to_kwargs(kwargs, kwargs["run"])
-
     super(Artifact, artifact).__init__(**kwargs)
 
 
@@ -692,8 +689,8 @@ def from_df(
         run=run,
         description=description,
         revises=revises,
-        _accessor="DataFrame",
-        type="dataset",
+        otype="DataFrame",
+        kind="dataset",
         **kwargs,
     )
     return artifact
@@ -719,8 +716,8 @@ def from_anndata(
         run=run,
         description=description,
         revises=revises,
-        _accessor="AnnData",
-        type="dataset",
+        otype="AnnData",
+        kind="dataset",
         **kwargs,
     )
     return artifact
@@ -744,8 +741,8 @@ def from_mudata(
         run=run,
         description=description,
         revises=revises,
-        _accessor="MuData",
-        type="dataset",
+        otype="MuData",
+        kind="dataset",
         **kwargs,
     )
     return artifact
@@ -879,13 +876,15 @@ def replace(
         if key_path.name != new_filename:
             self._clear_storagekey = self.key
             self.key = str(key_path.with_name(new_filename))
+            # update old key with the new one so that checks in record pass
+            self._old_key = self.key
             logger.warning(
                 f"replacing the file will replace key '{key_path}' with '{self.key}'"
                 f" and delete '{key_path}' upon `save()`"
             )
     else:
         old_storage = auto_storage_key_from_artifact(self)
-        is_dir = self.n_objects is not None
+        is_dir = self.n_files is not None
         new_storage = auto_storage_key_from_artifact_uid(
             self.uid, kwargs["suffix"], is_dir
         )
@@ -894,6 +893,8 @@ def replace(
             if self.key is not None:
                 new_key_path = PurePosixPath(self.key).with_suffix(kwargs["suffix"])
                 self.key = str(new_key_path)
+                # update old key with the new one so that checks in record pass
+                self._old_key = self.key
 
     self.suffix = kwargs["suffix"]
     self.size = kwargs["size"]
@@ -909,12 +910,21 @@ def replace(
     self._to_store = not check_path_in_storage
 
 
+inconsistent_state_msg = (
+    "Trying to read a folder artifact from an outdated version, "
+    "this can result in an incosistent state.\n"
+    "Read from the latest version: artifact.versions.filter(is_latest=True).one()"
+)
+
+
 # docstring handled through attach_func_to_class_method
 def open(
     self, mode: str = "r", is_run_input: bool | None = None
 ) -> (
     AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
 ):
+    if self._overwrite_versions and not self.is_latest:
+        raise ValueError(inconsistent_state_msg)
     # ignore empty suffix for now
     suffixes = ("", ".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma") + PYARROW_SUFFIXES
     if self.suffix not in suffixes:
@@ -993,6 +1003,9 @@ def _synchronize_cleanup_on_error(
 
 # docstring handled through attach_func_to_class_method
 def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
+    if self._overwrite_versions and not self.is_latest:
+        raise ValueError(inconsistent_state_msg)
+
     if hasattr(self, "_memory_rep") and self._memory_rep is not None:
         access_memory = self._memory_rep
     else:
@@ -1009,6 +1022,9 @@ def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
 
 # docstring handled through attach_func_to_class_method
 def cache(self, is_run_input: bool | None = None) -> Path:
+    if self._overwrite_versions and not self.is_latest:
+        raise ValueError(inconsistent_state_msg)
+
     filepath, cache_key = filepath_cache_key_from_artifact(
         self, using_key=settings._using_key
     )
@@ -1036,15 +1052,17 @@ def delete(
                 f"\n(2) If you want to delete the artifact in storage, please load the managing lamindb instance (uid={self.storage.instance_uid})."
                 f"\nThese are all managed storage locations of this instance:\n{Storage.filter(instance_uid=isettings.uid).df()}"
             )
-    # by default, we only move artifacts into the trash (visibility = -1)
-    trash_visibility = VisibilityChoice.trash.value
-    if self.visibility > trash_visibility and not permanent:
+    # by default, we only move artifacts into the trash (_branch_code = -1)
+    trash__branch_code = -1
+    if self._branch_code > trash__branch_code and not permanent:
         if storage is not None:
             logger.warning("moving artifact to trash, storage arg is ignored")
         # move to trash
-        self.visibility = trash_visibility
+        self._branch_code = trash__branch_code
         self.save()
-        logger.important(f"moved artifact to trash (visibility = {trash_visibility})")
+        logger.important(
+            f"moved artifact to trash (_branch_code = {trash__branch_code})"
+        )
         return
 
     # if the artifact is already in the trash
@@ -1070,8 +1088,21 @@ def delete(
             storage = False
         # only delete in storage if DB delete is successful
         # DB delete might error because of a foreign key constraint violated etc.
-        self._delete_skip_storage()
-        if self.key is None or self._key_is_virtual:
+        if self._overwrite_versions and self.is_latest:
+            # includes self
+            for version in self.versions.all():
+                _delete_skip_storage(version)
+        else:
+            self._delete_skip_storage()
+        # by default do not delete storage if deleting only a previous version
+        # and the underlying store is mutable
+        if self._overwrite_versions and not self.is_latest:
+            delete_in_storage = False
+            if storage:
+                logger.warning(
+                    "Storage argument is ignored; can't delete storage on an previous version"
+                )
+        elif self.key is None or self._key_is_virtual:
             # do not ask for confirmation also if storage is None
             delete_in_storage = storage is None or storage
         else:
@@ -1146,9 +1177,9 @@ def save(self, upload: bool | None = None, **kwargs) -> Artifact:
 
 
 def _save_skip_storage(file, **kwargs) -> None:
-    save_feature_sets(file)
+    save_staged__schemas_m2m(file)
     super(Artifact, file).save(**kwargs)
-    save_feature_set_links(file)
+    save_schema_links(file)
 
 
 @property  # type: ignore
@@ -1173,7 +1204,7 @@ def _cache_path(self) -> UPath:
 
 # docstring handled through attach_func_to_class_method
 def restore(self) -> None:
-    self.visibility = VisibilityChoice.default.value
+    self._branch_code = 1
     self.save()
 
 

lamindb/_can_curate.py

@@ -8,9 +8,10 @@ import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core import CanCurate, Record
 
-from ._from_values import _has_organism_field, _print_values, get_or_create_records
+from lamindb.models import CanCurate, Record
+
+from ._from_values import _format_values, _has_organism_field, get_or_create_records
 from ._record import _queryset, get_name_field
 from ._utils import attach_func_to_class_method
 from .core.exceptions import ValidationError
@@ -18,9 +19,9 @@ from .core.exceptions import ValidationError
 if TYPE_CHECKING:
     from django.db.models import QuerySet
     from lamin_utils._inspect import InspectResult
-    from lnschema_core.types import ListLike, StrField
 
     from lamindb._query_set import RecordList
+    from lamindb.base.types import ListLike, StrField
 
 
 # from_values doesn't apply for QuerySet or Manager
@@ -148,6 +149,11 @@ def _inspect(
     registry = queryset.model
     model_name = registry._meta.model.__name__
 
+    # do not inspect synonyms if the field is not name field
+    inspect_synonyms = True
+    if hasattr(registry, "_name_field") and field != registry._name_field:
+        inspect_synonyms = False
+
     # inspect in the DB
     result_db = inspect(
         df=_filter_query_based_on_organism(
@@ -156,19 +162,20 @@ def _inspect(
         identifiers=values,
         field=field,
         mute=mute,
+        inspect_synonyms=inspect_synonyms,
     )
     nonval = set(result_db.non_validated).difference(result_db.synonyms_mapper.keys())
 
-    if len(nonval) > 0 and registry.__get_schema_name__() == "bionty":
+    if len(nonval) > 0 and registry.__get_module_name__() == "bionty":
         try:
             bionty_result = registry.public(organism=organism, source=source).inspect(
-                values=nonval, field=field, mute=True
+                values=nonval, field=field, mute=True, inspect_synonyms=inspect_synonyms
             )
             bionty_validated = bionty_result.validated
             bionty_mapper = bionty_result.synonyms_mapper
             hint = False
             if len(bionty_validated) > 0 and not mute:
-                print_values = _print_values(bionty_validated)
+                print_values = _format_values(bionty_validated)
                 s = "" if len(bionty_validated) == 1 else "s"
                 labels = colors.yellow(f"{len(bionty_validated)} {model_name} term{s}")
                 logger.print(
@@ -178,7 +185,7 @@ def _inspect(
                 hint = True
 
             if len(bionty_mapper) > 0 and not mute:
-                print_values = _print_values(list(bionty_mapper.keys()))
+                print_values = _format_values(list(bionty_mapper.keys()))
                 s = "" if len(bionty_mapper) == 1 else "s"
                 labels = colors.yellow(f"{len(bionty_mapper)} {model_name} term{s}")
                 logger.print(
@@ -193,13 +200,13 @@ def _inspect(
                     f" {colors.italic('.from_values()')}"
                 )
 
-            nonval = bionty_result.non_validated
+            nonval = [i for i in bionty_result.non_validated if i not in bionty_mapper]
         # no bionty source is found
         except ValueError:
             logger.warning("no Bionty source found, skipping Bionty validation")
 
     if len(nonval) > 0 and not mute:
-        print_values = _print_values(list(nonval))
+        print_values = _format_values(list(nonval))
         s = "" if len(nonval) == 1 else "s"
         labels = colors.red(f"{len(nonval)} term{s}")
         logger.print(f"   couldn't validate {labels}: {colors.red(print_values)}")
@@ -383,6 +390,8 @@ def _standardize(
         organism = (
             organism_record.name if organism_record is not None else organism_record
         )
+    else:
+        organism = None
 
     # only perform synonym mapping if field is the name field
     if hasattr(registry, "_name_field") and field != registry._name_field:
@@ -429,7 +438,7 @@ def _standardize(
             return result
 
     # map synonyms in Bionty
-    if registry.__get_schema_name__() == "bionty" and public_aware:
+    if registry.__get_module_name__() == "bionty" and public_aware:
         mapper = {}
         if return_mapper:
             mapper = std_names_db