lamindb 0.76.2__py3-none-any.whl → 0.76.4__py3-none-any.whl

lamindb/__init__.py

@@ -1,7 +1,6 @@
 """A data framework for biology.
 
-Records
-=======
+Core registries.
 
 .. autosummary::
    :toctree: .
@@ -17,20 +16,18 @@ Records
    FeatureSet
    Param
 
-Key functionality
-=================
+Key functionality.
 
 .. autosummary::
    :toctree: .
 
    context
    connect
-   Curate
+   Curator
    view
    save
 
-Modules & settings
-==================
+Modules and settings.
 
 .. autosummary::
    :toctree: .
@@ -44,7 +41,7 @@ Modules & settings
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.2"
+__version__ = "0.76.4"
 
 import os as _os
 
@@ -94,7 +91,7 @@ if _check_instance_setup(from_lamindb=True):
         _ulabel,
         integrations,
     )
-    from ._curate import Curate
+    from ._curate import Curator
     from ._save import save
     from ._view import view
     from .core._context import context
@@ -110,6 +107,7 @@ if _check_instance_setup(from_lamindb=True):
 
     track = context.track  # backward compat
     finish = context.finish  # backward compat
+    Curate = Curator  # backward compat
     settings.__doc__ = """Global :class:`~lamindb.core.Settings`."""
     context.__doc__ = """Global :class:`~lamindb.core.Context`."""
     from django.db.models import Q

lamindb/_artifact.py

@@ -366,11 +366,6 @@ def get_artifact_kwargs_from_data(
     else:
         storage = default_storage
 
-    # for now comment out this error to allow creating new versions of stores
-    # in the default folder (.lamindb)
-    #    if key is not None and key.startswith(AUTO_KEY_PREFIX):
-    #        raise ValueError(f"Key cannot start with {AUTO_KEY_PREFIX}")
-
     log_storage_hint(
         check_path_in_storage=check_path_in_storage,
         storage=storage,
@@ -542,6 +537,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         else VisibilityChoice.default.value
     )
     format = kwargs.pop("format") if "format" in kwargs else None
+    _is_internal_call = kwargs.pop("_is_internal_call", False)
     skip_check_exists = (
         kwargs.pop("skip_check_exists") if "skip_check_exists" in kwargs else False
     )
@@ -575,13 +571,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError(
             f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
         )
-
-    provisional_uid, revises = create_uid(revises=revises, version=version)
     if revises is not None:
         if not isinstance(revises, Artifact):
             raise TypeError("`revises` has to be of type `Artifact`")
         if description is None:
             description = revises.description
+    if key is not None and AUTO_KEY_PREFIX in key:
+        raise ValueError(
+            f"Do not pass key that contains a managed storage path in `{AUTO_KEY_PREFIX}`"
+        )
+    # below is for internal calls that require defining the storage location
+    # ahead of constructing the Artifact
+    if isinstance(data, (str, Path)) and AUTO_KEY_PREFIX in str(data):
+        if _is_internal_call:
+            is_automanaged_path = True
+            user_provided_key = key
+            key = None
+        else:
+            raise ValueError(
+                f"Do not pass path inside the `{AUTO_KEY_PREFIX}` directory."
+            )
+    else:
+        is_automanaged_path = False
+    provisional_uid, revises = create_uid(revises=revises, version=version)
     kwargs_or_artifact, privates = get_artifact_kwargs_from_data(
         data=data,
         key=key,
@@ -609,16 +621,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    if data is not None:
+        artifact._local_filepath = privates["local_filepath"]
+        artifact._cloud_filepath = privates["cloud_filepath"]
+        artifact._memory_rep = privates["memory_rep"]
+        artifact._to_store = not privates["check_path_in_storage"]
+
+    if is_automanaged_path and _is_internal_call:
+        kwargs["_key_is_virtual"] = True
+        assert AUTO_KEY_PREFIX in kwargs["key"]  # noqa: S101
+        uid = kwargs["key"].replace(AUTO_KEY_PREFIX, "").replace(kwargs["suffix"], "")
+        kwargs["key"] = user_provided_key
+        if revises is not None:
+            assert uid.startswith(revises.stem_uid)  # noqa: S101
+        if len(uid) == 16:
+            if revises is None:
+                uid += "0000"
+            else:
+                uid, revises = create_uid(revises=revises, version=version)
+        kwargs["uid"] = uid
+
     # only set key now so that we don't do a look-up on it in case revises is passed
     if revises is not None:
         kwargs["key"] = revises.key
-    # in case we have a new version of a folder with a different hash, print a
-    # warning that the old version can't be recovered
-    if revises is not None and revises.n_objects is not None and revises.n_objects > 1:
-        logger.warning(
-            f"artifact version {version} will _update_ the state of folder {revises.path} - "
-            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `revises`"
-        )
 
     kwargs["type"] = type
     kwargs["version"] = version
@@ -637,12 +662,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
 
     add_transform_to_kwargs(kwargs, kwargs["run"])
 
-    if data is not None:
-        artifact._local_filepath = privates["local_filepath"]
-        artifact._cloud_filepath = privates["cloud_filepath"]
-        artifact._memory_rep = privates["memory_rep"]
-        artifact._to_store = not privates["check_path_in_storage"]
-
     super(Artifact, artifact).__init__(**kwargs)
 
 
@@ -937,10 +956,9 @@ def open(
                 if self.hash != hash:
                     from ._record import init_self_from_db
 
-                    logger.warning(
-                        "The hash of the tiledbsoma store has changed, creating a new version of the artifact."
-                    )
-                    new_version = Artifact(filepath, revises=self).save()
+                    new_version = Artifact(
+                        filepath, revises=self, _is_internal_call=True
+                    ).save()
                     init_self_from_db(self, new_version)
 
                     if localpath != filepath and localpath.exists():
@@ -1168,3 +1186,4 @@ Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
 Artifact.backed = backed
+Artifact.view_lineage = HasFeatures.view_lineage

lamindb/_can_validate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable, Literal
+from typing import TYPE_CHECKING, Literal
 
 import lamindb_setup as ln_setup
 import numpy as np
@@ -79,6 +79,19 @@ def _check_organism_db(organism: Record, using_key: str | None):
                 )
 
 
+def _concat_lists(values: ListLike) -> list[str]:
+    """Concatenate a list of lists of strings into a single list."""
+    if len(values) > 0 and isinstance(values, (list, pd.Series)):
+        try:
+            if isinstance(values[0], list):
+                if isinstance(values, pd.Series):
+                    values = values.tolist()
+                values = sum([v for v in values if isinstance(v, list)], [])
+        except KeyError:
+            pass
+    return values
+
+
 def _inspect(
     cls,
     values: ListLike,
@@ -94,6 +107,7 @@ def _inspect(
 
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     queryset = _queryset(cls, using_key)
@@ -184,6 +198,7 @@ def _validate(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
 
@@ -229,7 +244,7 @@ def _validate(
 @doc_args(CanValidate.standardize.__doc__)
 def standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -295,7 +310,7 @@ def remove_synonym(self, synonym: str | ListLike):
 
 def _standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -315,6 +330,7 @@ def _standardize(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     return_field = get_name_field(
@@ -416,7 +432,7 @@ def _standardize(
 
 
 def _add_or_remove_synonyms(
-    synonym: str | Iterable,
+    synonym: str | ListLike,
     record: Record,
     action: Literal["add", "remove"],
     force: bool = False,

lamindb/_curate.py

@@ -84,10 +84,34 @@ class CurateLookup:
             return colors.warning("No fields are found!")
 
 
-class DataFrameCurator:
+class BaseCurator:
+    """Curate a dataset."""
+
+    def validate(self) -> bool:
+        """Validate dataset.
+
+        Returns:
+            Boolean indicating whether the dataset is validated.
+        """
+        pass
+
+    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+        """Save the dataset as artifact.
+
+        Args:
+            description: Description of the DataFrame object.
+            **kwargs: Object level metadata.
+
+        Returns:
+            A saved artifact record.
+        """
+        pass
+
+
+class DataFrameCurator(BaseCurator):
     """Curation flow for a DataFrame object.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
 
     Args:
         df: The DataFrame object to curate.
@@ -101,7 +125,7 @@ class DataFrameCurator:
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_df(
+        >>> curate = ln.Curator.from_df(
         ...     df,
         ...     categoricals={
         ...         "cell_type_ontology_id": bt.CellType.ontology_id,
@@ -120,6 +144,7 @@ class DataFrameCurator:
         organism: str | None = None,
         sources: dict[str, Record] | None = None,
         exclude: dict | None = None,
+        check_valid_keys: bool = True,
     ) -> None:
         from lamindb.core._settings import settings
 
@@ -139,6 +164,8 @@ class DataFrameCurator:
             exclude = {}
         self._exclude = exclude
         self._non_validated = None
+        if check_valid_keys:
+            self._check_valid_keys()
         self._save_columns()
 
     @property
@@ -167,14 +194,25 @@ class DataFrameCurator:
             using_key=using_key or self._using_key,
         )
 
+    def _check_valid_keys(self, extra: set = None) -> None:
+        if extra is None:
+            extra = set()
+        for name, d in {
+            "categoricals": self._fields,
+            "sources": self._sources,
+            "exclude": self._exclude,
+        }.items():
+            if not isinstance(d, dict):
+                raise TypeError(f"{name} must be a dictionary!")
+            valid_keys = set(self._df.columns) | {"columns"} | extra
+            nonval_keys = [key for key in d.keys() if key not in valid_keys]
+            if len(nonval_keys) > 0:
+                raise ValueError(
+                    f"the following keys passed to {name} are not allowed: {nonval_keys}"
+                )
+
     def _save_columns(self, validated_only: bool = True, **kwargs) -> None:
         """Save column name records."""
-        missing_columns = set(self.fields.keys()) - set(self._df.columns)
-        if missing_columns:
-            raise ValueError(
-                f"Columns {missing_columns} are not found in the data object!"
-            )
-
         # Always save features specified as the fields keys
         update_registry(
             values=list(self.fields.keys()),
@@ -184,6 +222,7 @@ class DataFrameCurator:
             using_key=self._using_key,
             validated_only=False,
             source=self._sources.get("columns"),
+            exclude=self._exclude.get("columns"),
             **kwargs,
         )
 
@@ -199,6 +238,7 @@ class DataFrameCurator:
                 validated_only=validated_only,
                 df=self._df,  # Get the Feature type from df
                 source=self._sources.get("columns"),
+                exclude=self._exclude.get("columns"),
                 warning=False,  # Do not warn about missing columns, just an info message
                 **kwargs,
             )
@@ -251,6 +291,7 @@ class DataFrameCurator:
                 using_key=self._using_key,
                 validated_only=validated_only,
                 source=self._sources.get(categorical),
+                exclude=self._exclude.get(categorical),
                 **kwargs,
             )
 
@@ -330,9 +371,11 @@ class DataFrameCurator:
 class AnnDataCurator(DataFrameCurator):
     """Curation flow for ``AnnData``.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
+
+    Note that if genes are removed from the AnnData object, the object should be recreated using :meth:`~lamindb.Curator.from_anndata`.
 
-    Note that if genes are removed from the AnnData object, the object should be recreated using :meth:`~lamindb.Curate.from_anndata`.
+    See :doc:`docs:cellxgene-curate` for instructions on how to curate against a specific cellxgene schema version.
 
     Args:
         data: The AnnData object or an AnnData-like path.
@@ -346,7 +389,7 @@ class AnnDataCurator(DataFrameCurator):
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_anndata(
+        >>> curate = ln.Curator.from_anndata(
         ...     adata,
         ...     var_index=bt.Gene.ensembl_gene_id,
         ...     categoricals={
@@ -397,8 +440,10 @@ class AnnDataCurator(DataFrameCurator):
             organism=organism,
             sources=sources,
             exclude=exclude,
+            check_valid_keys=False,
         )
         self._obs_fields = categoricals
+        self._check_valid_keys(extra={"var_index"})
 
     @property
     def var_index(self) -> FieldAttr:
@@ -437,6 +482,7 @@ class AnnDataCurator(DataFrameCurator):
             validated_only=validated_only,
             organism=organism,
             source=self._sources.get("var_index"),
+            exclude=self._exclude.get("var_index"),
         )
 
     def _update_registry_all(self, validated_only: bool = True, **kwargs):
@@ -536,10 +582,10 @@ class AnnDataCurator(DataFrameCurator):
 class MuDataCurator:
     """Curation flow for a ``MuData`` object.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
 
     Note that if genes or other measurements are removed from the MuData object,
-    the object should be recreated using :meth:`~lamindb.Curate.from_mudata`.
+    the object should be recreated using :meth:`~lamindb.Curator.from_mudata`.
 
     Args:
         mdata: The MuData object to curate.
@@ -556,7 +602,7 @@ class MuDataCurator:
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_mudata(
+        >>> curate = ln.Curator.from_mudata(
         ...     mdata,
         ...     var_index={
         ...         "rna": bt.Gene.ensembl_gene_id,
@@ -603,6 +649,7 @@ class MuDataCurator:
                 verbosity=verbosity,
                 sources=self._sources.get(modality),
                 exclude=self._exclude.get(modality),
+                check_valid_keys=False,
                 **self._kwargs,
             )
             for modality in self._modalities
@@ -641,6 +688,7 @@ class MuDataCurator:
             validated_only=validated_only,
             dtype="number",
             source=self._sources.get(modality, {}).get("var_index"),
+            exclude=self._exclude.get(modality, {}).get("var_index"),
             **kwargs,
         )
 
@@ -704,6 +752,7 @@ class MuDataCurator:
             validated_only=False,
             df=self._mdata[modality].obs,
             source=self._sources.get(modality, {}).get("columns"),
+            exclude=self._exclude.get(modality, {}).get("columns"),
             **self._kwargs,  # type: ignore
             **kwargs,
         )
@@ -789,7 +838,8 @@ class MuDataCurator:
                 field=var_field,
                 key=f"{modality}_var_index",
                 using_key=self._using_key,
-                exclude=self._exclude.get(f"{modality}_var_index"),
+                source=self._sources.get(modality, {}).get("var_index"),
+                exclude=self._exclude.get(modality, {}).get("var_index"),
                 **self._kwargs,  # type: ignore
             )
             validated_var &= is_validated_var
@@ -846,19 +896,19 @@ class MuDataCurator:
         return self._artifact
 
 
-class Curate:
-    """Curation flow.
+class Curator(BaseCurator):
+    """Dataset curator.
 
     Data curation entails accurately labeling datasets with standardized metadata
     to facilitate data integration, interpretation and analysis.
 
     The curation flow has several steps:
 
-    1. Create a :class:`Curate` object corresponding to the object type that you want to curate:
+    1. Instantiate `Curator` from one of the following dataset objects:
 
-    - :meth:`~lamindb.Curate.from_df`
-    - :meth:`~lamindb.Curate.from_anndata`
-    - :meth:`~lamindb.Curate.from_mudata`
+    - :meth:`~lamindb.Curator.from_df`
+    - :meth:`~lamindb.Curator.from_anndata`
+    - :meth:`~lamindb.Curator.from_mudata`
 
     During object creation, any passed categoricals found in the object will be saved.
 
@@ -867,7 +917,7 @@ class Curate:
     - Values that can successfully validated and already exist in the registry.
     - Values which are new and not yet validated or potentially problematic values.
 
-    3. Determine how to handle validated and unvalidated values:
+    3. Determine how to handle validated and non-validated values:
 
     - Validated values not yet in the registry can be automatically registered using :meth:`~lamindb.core.DataFrameCurator.add_validated_from`.
     - Valid and new values can be registered using :meth:`~lamindb.core.DataFrameCurator.add_new_from`.
@@ -982,10 +1032,22 @@ def standardize_and_inspect(
     field: FieldAttr,
     registry: type[Record],
     standardize: bool = False,
+    exclude: str | list | None = None,
     **kwargs,
 ):
     """Standardize and inspect values using a registry."""
-    filter_kwargs = get_current_filter_kwargs(registry, kwargs)
+    # inspect exclude values in the default instance
+    values = list(values)
+    include_validated = []
+    if exclude is not None:
+        exclude = [exclude] if isinstance(exclude, str) else exclude
+        exclude = [i for i in exclude if i in values]
+        if len(exclude) > 0:
+            # exclude values are validated without source and organism
+            inspect_result_exclude = registry.inspect(exclude, field=field, mute=True)
+            # if exclude values are validated, remove them from the values
+            values = [i for i in values if i not in inspect_result_exclude.validated]
+            include_validated = inspect_result_exclude.validated
 
     if standardize:
         if hasattr(registry, "standardize") and hasattr(
@@ -993,11 +1055,17 @@ def standardize_and_inspect(
             "synonyms",  # https://github.com/laminlabs/lamindb/issues/1685
         ):
             standardized_values = registry.standardize(
-                values, field=field, mute=True, **filter_kwargs
+                values, field=field, mute=True, **kwargs
             )
             values = standardized_values
 
-    return registry.inspect(values, field=field, mute=True, **filter_kwargs)
+    inspect_result = registry.inspect(values, field=field, mute=True, **kwargs)
+    inspect_result._validated += include_validated
+    inspect_result._non_validated = [
+        i for i in inspect_result.non_validated if i not in include_validated
+    ]
+
+    return inspect_result
 
 
 def check_registry_organism(registry: Record, organism: str | None = None) -> dict:
@@ -1049,35 +1117,32 @@ def validate_categories(
         logger.indent = "   "
 
     registry = field.field.model
+
     kwargs = check_registry_organism(registry, organism)
     kwargs.update({"source": source} if source else {})
+    kwargs_current = get_current_filter_kwargs(registry, kwargs)
 
     # inspect the default instance
-    if exclude is not None:
-        exclude = [exclude] if isinstance(exclude, str) else exclude
-        # exclude values are validated without source and organism
-        inspect_result = registry.inspect(exclude, field=field, mute=True)
-        # if exclude values are validated, remove them from the values
-        values = [i for i in values if i not in inspect_result.validated]
-
     inspect_result = standardize_and_inspect(
         values=values,
         field=field,
         registry=registry,
         standardize=standardize,
-        **kwargs,
+        exclude=exclude,
+        **kwargs_current,
     )
     non_validated = inspect_result.non_validated
 
+    # inspect the using instance
     values_validated = []
     if using_key is not None and using_key != "default" and non_validated:
         registry_using = get_registry_instance(registry, using_key)
-        # inspect the using instance
         inspect_result = standardize_and_inspect(
             values=non_validated,
             field=field,
             registry=registry_using,
             standardize=standardize,
+            exclude=exclude,
             **kwargs,
         )
         non_validated = inspect_result.non_validated
@@ -1091,7 +1156,7 @@ def validate_categories(
             public_records = registry.from_values(
                 non_validated,
                 field=field,
-                **get_current_filter_kwargs(registry, kwargs),
+                **kwargs_current,
             )
             values_validated += [getattr(r, field.field.name) for r in public_records]
         finally:
@@ -1111,9 +1176,13 @@ def validate_categories(
     non_validated = [i for i in non_validated if i not in values_validated]
     n_non_validated = len(non_validated)
     if n_non_validated == 0:
-        logger.indent = ""
-        logger.success(f"{key} is validated against {colors.italic(model_field)}")
-        return True, []
+        if n_validated == 0:
+            logger.indent = ""
+            logger.success(f"{key} is validated against {colors.italic(model_field)}")
+            return True, []
+        else:
+            # validated values still need to be saved to the current instance
+            return False, []
     else:
         are = "are" if n_non_validated > 1 else "is"
         print_values = _print_values(non_validated)
@@ -1138,6 +1207,9 @@ def validate_categories_in_df(
     **kwargs,
 ) -> tuple[bool, dict]:
     """Validate categories in DataFrame columns using LaminDB registries."""
+    if not fields:
+        return True, {}
+
     if sources is None:
         sources = {}
     validated = True
@@ -1270,6 +1342,7 @@ def update_registry(
     source: Record | None = None,
     standardize: bool = True,
     warning: bool = True,
+    exclude: str | list | None = None,
     **kwargs,
 ) -> None:
     """Save features or labels records in the default instance from the using_key instance.
@@ -1329,7 +1402,8 @@ def update_registry(
             field=field,
             registry=registry,
             standardize=standardize,
-            **filter_kwargs,
+            exclude=exclude,
+            **filter_kwargs_current,
         )
         if not inspect_result_current.non_validated:
             all_labels = registry.from_values(
@@ -1348,6 +1422,7 @@ def update_registry(
             inspect_result_current.non_validated,
             field=field,
             using_key=using_key,
+            exclude=exclude,
             **filter_kwargs,
         )
 
@@ -1467,6 +1542,7 @@ def update_registry_from_using_instance(
     field: FieldAttr,
     using_key: str | None = None,
     standardize: bool = False,
+    exclude: str | list | None = None,
     **kwargs,
 ) -> tuple[list[str], list[str]]:
     """Save features or labels records from the using_key instance.
@@ -1492,6 +1568,7 @@ def update_registry_from_using_instance(
             field=field,
             registry=registry_using,
             standardize=standardize,
+            exclude=exclude,
             **kwargs,
         )
         labels_using = registry_using.filter(
@@ -1519,3 +1596,6 @@ def _save_organism(name: str):  # pragma: no cover
             )
         organism.save()
     return organism
+
+
+Curate = Curator  # backward compat