lamindb 0.76.12__py3-none-any.whl → 0.76.14__py3-none-any.whl

lamindb/_curate.py

@@ -34,21 +34,21 @@ class CurateLookup:
         categoricals: dict[str, FieldAttr],
         slots: dict[str, FieldAttr] = None,
         using_key: str | None = None,
+        public: bool = False,
     ) -> None:
         if slots is None:
             slots = {}
         self._fields = {**categoricals, **slots}
         self._using_key = None if using_key == "default" else using_key
         self._using_key_name = self._using_key or ln_setup.settings.instance.slug
-        debug_message = (
-            f"Lookup objects from the " f"{colors.italic(self._using_key_name)}"
-        )
+        self._public = public
+        debug_message = f"Lookup objects from {colors.italic(self._using_key_name)}"
         logger.debug(debug_message)
 
     def __getattr__(self, name):
         if name in self._fields:
             registry = self._fields[name].field.model
-            if self._using_key == "public":
+            if self._public and hasattr(registry, "public"):
                 return registry.public().lookup()
             else:
                 return get_registry_instance(registry, self._using_key).lookup()
@@ -59,7 +59,7 @@ class CurateLookup:
     def __getitem__(self, name):
         if name in self._fields:
             registry = self._fields[name].field.model
-            if self._using_key == "public":
+            if self._public and hasattr(registry, "public"):
                 return registry.public().lookup()
             else:
                 return get_registry_instance(registry, self._using_key).lookup()
@@ -75,12 +75,14 @@ class CurateLookup:
             getitem_keys = "\n ".join(
                 [str([key]) for key in self._fields if not key.isidentifier()]
             )
+            ref = "public" if self._public else self._using_key_name
             return (
-                f"Lookup objects from the {colors.italic(self._using_key_name)}:\n "
+                f"Lookup objects from the {colors.italic(ref)}:\n "
                 f"{colors.green(getattr_keys)}\n "
-                f"{colors.green(getitem_keys)}\n\n"
-                "Example:\n    → categories = validator.lookup().cell_type\n"
-                "    → categories.alveolar_type_1_fibroblast_cell"
+                f"{colors.green(getitem_keys)}\n"
+                "Example:\n    → categories = validator.lookup()['cell_type']\n"
+                "    → categories.alveolar_type_1_fibroblast_cell\n\n"
+                "To look up public ontologies, use .lookup(public=True)"
             )
         else:  # pragma: no cover
             return colors.warning("No fields are found!")
@@ -97,12 +99,20 @@ class BaseCurator:
         """
         pass
 
-    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+    def save_artifact(
+        self,
+        description: str | None = None,
+        key: str | None = None,
+        revises: Artifact | None = None,
+        run: Run | None = None,
+    ) -> Artifact:
         """Save the dataset as artifact.
 
         Args:
-            description: Description of the DataFrame object.
-            **kwargs: Object level metadata.
+            description: `str | None = None` A description of the DataFrame object.
+            key: `str | None = None` A path-like key to reference artifact in default storage, e.g., `"myfolder/myfile.fcs"`. Artifacts with the same key form a revision family.
+            revises: `Artifact | None = None` Previous version of the artifact. Triggers a revision.
+            run: `Run | None = None` The run that creates the artifact.
 
         Returns:
             A saved artifact record.
@@ -182,7 +192,9 @@ class DataFrameCurator(BaseCurator):
         """Return the columns fields to validate against."""
         return self._fields
 
-    def lookup(self, using_key: str | None = None) -> CurateLookup:
+    def lookup(
+        self, using_key: str | None = None, public: bool = False
+    ) -> CurateLookup:
         """Lookup categories.
 
         Args:
@@ -194,6 +206,7 @@ class DataFrameCurator(BaseCurator):
             categoricals=self._fields,
             slots={"columns": self._columns_field},
             using_key=using_key or self._using_key,
+            public=public,
         )
 
     def _check_valid_keys(self, extra: set = None) -> None:
@@ -245,16 +258,6 @@ class DataFrameCurator(BaseCurator):
                 **kwargs,
             )
 
-    def add_validated_from(self, key: str, organism: str | None = None):
-        """Add validated categories.
-
-        Args:
-            key: The key referencing the slot in the DataFrame.
-            organism: The organism name.
-        """
-        self._kwargs.update({"organism": organism} if organism else {})
-        self._update_registry(key, validated_only=True, **self._kwargs)
-
     def add_new_from(self, key: str, organism: str | None = None, **kwargs):
         """Add validated & new categories.
 
@@ -300,7 +303,7 @@ class DataFrameCurator(BaseCurator):
     def _update_registry_all(self, validated_only: bool = True, **kwargs):
         """Save labels for all features."""
         for name in self.fields.keys():
-            logger.info(f"saving labels for '{name}'")
+            logger.info(f"saving validated records of '{name}'")
             self._update_registry(name, validated_only=validated_only, **kwargs)
 
     def validate(self, organism: str | None = None) -> bool:
@@ -313,6 +316,10 @@ class DataFrameCurator(BaseCurator):
             Whether the DataFrame is validated.
         """
         self._kwargs.update({"organism": organism} if organism else {})
+
+        # add all validated records to the current instance
+        self._update_registry_all()
+
         self._validated, self._non_validated = validate_categories_in_df(  # type: ignore
             self._df,
             fields=self.fields,
@@ -323,12 +330,20 @@ class DataFrameCurator(BaseCurator):
         )
         return self._validated
 
-    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+    def save_artifact(
+        self,
+        description: str | None = None,
+        key: str | None = None,
+        revises: Artifact | None = None,
+        run: Run | None = None,
+    ) -> Artifact:
         """Save the validated DataFrame and metadata.
 
         Args:
-            description: Description of the DataFrame object.
-            **kwargs: Object level metadata.
+            description: `str | None = None` Description of the DataFrame object.
+            key: `str | None = None` A path-like key to reference artifact in default storage, e.g., `"myfolder/myfile.fcs"`. Artifacts with the same key form a revision family.
+            revises: `Artifact | None = None` Previous version of the artifact. Triggers a revision.
+            run: `Run | None = None` The run that creates the artifact.
 
         Returns:
             A saved artifact record.
@@ -344,15 +359,18 @@ class DataFrameCurator(BaseCurator):
         verbosity = settings.verbosity
         try:
             settings.verbosity = "warning"
-            # save all validated records to the current instance
-            self.add_validated_from("all")
+            if not self._validated:
+                # save all validated records to the current instance
+                self._update_registry_all()
 
             self._artifact = save_artifact(
                 self._df,
                 description=description,
                 fields=self.fields,
                 columns_field=self._columns_field,
-                **kwargs,
+                key=key,
+                revises=revises,
+                run=run,
                 **self._kwargs,
             )
         finally:
@@ -457,7 +475,9 @@ class AnnDataCurator(DataFrameCurator):
         """Return the obs fields to validate against."""
         return self._obs_fields
 
-    def lookup(self, using_key: str | None = None) -> CurateLookup:
+    def lookup(
+        self, using_key: str | None = None, public: bool = False
+    ) -> CurateLookup:
         """Lookup categories.
 
         Args:
@@ -469,6 +489,7 @@ class AnnDataCurator(DataFrameCurator):
             categoricals=self._obs_fields,
             slots={"columns": self._columns_field, "var_index": self._var_field},
             using_key=using_key or self._using_key,
+            public=public,
         )
 
     def _save_from_var_index(
@@ -479,7 +500,7 @@ class AnnDataCurator(DataFrameCurator):
             values=list(self._adata.var.index),
             field=self.var_index,
             key="var_index",
-            save_function="add_new_from_var_index",
+            save_function=".add_new_from_var_index()",
             using_key=self._using_key,
             validated_only=validated_only,
             organism=organism,
@@ -487,14 +508,13 @@ class AnnDataCurator(DataFrameCurator):
             exclude=self._exclude.get("var_index"),
         )
 
-    def _update_registry_all(self, validated_only: bool = True, **kwargs):
+    def _update_registry_all(self):
         """Save labels for all features."""
-        for name in self.fields.keys():
-            logger.info(f"saving labels for '{name}'")
-            if name == "var_index":
-                self._save_from_var_index(validated_only=validated_only, **kwargs)
-            else:
-                self._update_registry(name, validated_only=validated_only, **kwargs)
+        logger.info("saving validated records of 'var_index'")
+        self._save_from_var_index(validated_only=True, **self._kwargs)
+        for name in self._obs_fields.keys():
+            logger.info(f"saving validated terms of '{name}'")
+            self._update_registry(name, validated_only=True, **self._kwargs)
 
     def add_new_from_var_index(self, organism: str | None = None, **kwargs):
         """Update variable records.
@@ -506,15 +526,6 @@ class AnnDataCurator(DataFrameCurator):
         self._kwargs.update({"organism": organism} if organism else {})
         self._save_from_var_index(validated_only=False, **self._kwargs, **kwargs)
 
-    def add_validated_from_var_index(self, organism: str | None = None):
-        """Add validated variable records.
-
-        Args:
-            organism: The organism name.
-        """
-        self._kwargs.update({"organism": organism} if organism else {})
-        self._save_from_var_index(validated_only=True, **self._kwargs)
-
     def validate(self, organism: str | None = None) -> bool:
         """Validate categories.
 
@@ -530,6 +541,9 @@ class AnnDataCurator(DataFrameCurator):
                 f"validating metadata using registries of instance {colors.italic(self._using_key)}"
             )
 
+        # add all validated records to the current instance
+        self._update_registry_all()
+
         validated_var, non_validated_var = validate_categories(
             self._adata.var.index,
             field=self._var_field,
@@ -554,30 +568,49 @@ class AnnDataCurator(DataFrameCurator):
         self._validated = validated_var and validated_obs
         return self._validated
 
-    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+    def save_artifact(
+        self,
+        description: str | None = None,
+        key: str | None = None,
+        revises: Artifact | None = None,
+        run: Run | None = None,
+    ) -> Artifact:
         """Save the validated ``AnnData`` and metadata.
 
         Args:
-            description: Description of the ``AnnData`` object.
-            **kwargs: Object level metadata.
+            description: `str | None = None` A description of the ``AnnData`` object.
+            key: `str | None = None` A path-like key to reference artifact in default storage, e.g., `"myfolder/myfile.fcs"`. Artifacts with the same key form a revision family.
+            revises: `Artifact | None = None` Previous version of the artifact. Triggers a revision.
+            run: `Run | None = None` The run that creates the artifact.
 
         Returns:
             A saved artifact record.
         """
+        from lamindb.core._settings import settings
+
         if not self._validated:
             self.validate()
             if not self._validated:
                 raise ValidationError("Dataset does not validate. Please curate.")
-
-        self._artifact = save_artifact(
-            self._data,
-            adata=self._adata,
-            description=description,
-            columns_field=self.var_index,
-            fields=self.categoricals,
-            **self._kwargs,
-            **kwargs,
-        )
+        verbosity = settings.verbosity
+        try:
+            settings.verbosity = "warning"
+            if not self._validated:
+                # save all validated records to the current instance
+                self._update_registry_all()
+            self._artifact = save_artifact(
+                self._data,
+                adata=self._adata,
+                description=description,
+                columns_field=self.var_index,
+                fields=self.categoricals,
+                key=key,
+                revises=revises,
+                run=run,
+                **self._kwargs,
+            )
+        finally:
+            settings.verbosity = verbosity
         return self._artifact
 
 
@@ -656,10 +689,6 @@ class MuDataCurator:
             )
             for modality in self._modalities
         }
-        for modality in self._var_fields.keys():
-            self._save_from_var_index_modality(
-                modality=modality, validated_only=True, **self._kwargs
-            )
 
     @property
     def var_index(self) -> FieldAttr:
@@ -685,7 +714,7 @@ class MuDataCurator:
             values=list(self._mdata[modality].var.index),
             field=self._var_fields[modality],
             key="var_index",
-            save_function="add_new_from_var_index",
+            save_function=f'.add_new_from_var_index("{modality}")',
             using_key=self._using_key,
             validated_only=validated_only,
             dtype="number",
@@ -712,7 +741,9 @@ class MuDataCurator:
                 obs_fields["obs"][k] = v
         return obs_fields
 
-    def lookup(self, using_key: str | None = None) -> CurateLookup:
+    def lookup(
+        self, using_key: str | None = None, public: bool = False
+    ) -> CurateLookup:
         """Lookup categories.
 
         Args:
@@ -727,6 +758,7 @@ class MuDataCurator:
                 **{f"{k}_var_index": v for k, v in self._var_fields.items()},
             },
             using_key=using_key or self._using_key,
+            public=public,
         )
 
     def add_new_from_columns(
@@ -774,33 +806,14 @@ class MuDataCurator:
             modality=modality, validated_only=False, **self._kwargs, **kwargs
         )
 
-    def add_validated_from_var_index(self, modality: str, organism: str | None = None):
-        """Add validated variable records.
-
-        Args:
-            modality: The modality name.
-            organism: The organism name.
-        """
-        self._kwargs.update({"organism": organism} if organism else {})
-        self._save_from_var_index_modality(
-            modality=modality, validated_only=True, **self._kwargs
-        )
-
-    def add_validated_from(
-        self, key: str, modality: str | None = None, organism: str | None = None
-    ):
-        """Add validated categories.
-
-        Args:
-            key: The key referencing the slot in the DataFrame.
-            modality: The modality name.
-            organism: The organism name.
-        """
-        self._kwargs.update({"organism": organism} if organism else {})
-        modality = modality or "obs"
-        if modality in self._df_annotators:
-            df_annotator = self._df_annotators[modality]
-            df_annotator.add_validated_from(key=key, **self._kwargs)
+    def _update_registry_all(self):
+        """Update all registries."""
+        for modality in self._var_fields.keys():
+            self._save_from_var_index_modality(
+                modality=modality, validated_only=True, **self._kwargs
+            )
+        for _, df_annotator in self._df_annotators.items():
+            df_annotator._update_registry_all(validated_only=True, **self._kwargs)
 
     def add_new_from(
         self,
@@ -827,11 +840,22 @@ class MuDataCurator:
 
     def validate(self, organism: str | None = None) -> bool:
         """Validate categories."""
+        from lamindb.core._settings import settings
+
         self._kwargs.update({"organism": organism} if organism else {})
         if self._using_key is not None and self._using_key != "default":
             logger.important(
                 f"validating metadata using registries of instance {colors.italic(self._using_key)}"
             )
+
+        # add all validated records to the current instance
+        verbosity = settings.verbosity
+        try:
+            settings.verbosity = "error"
+            self._update_registry_all()
+        finally:
+            settings.verbosity = verbosity
+
         validated_var = True
         non_validated_var_modality = {}
         for modality, var_field in self._var_fields.items():
@@ -842,6 +866,7 @@ class MuDataCurator:
                 using_key=self._using_key,
                 source=self._sources.get(modality, {}).get("var_index"),
                 exclude=self._exclude.get(modality, {}).get("var_index"),
+                validated_hint_print=f'.add_validated_from_var_index("{modality}")',
                 **self._kwargs,  # type: ignore
             )
             validated_var &= is_validated_var
@@ -874,56 +899,75 @@ class MuDataCurator:
         self._validated = validated_var and validated_obs
         return self._validated
 
-    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+    def save_artifact(
+        self,
+        description: str | None = None,
+        key: str | None = None,
+        revises: Artifact | None = None,
+        run: Run | None = None,
+    ) -> Artifact:
         """Save the validated ``MuData`` and metadata.
 
         Args:
-            description: Description of the ``MuData`` object.
-            **kwargs: Object level metadata.
+            description: `str | None = None` A description of the ``MuData`` object.
+            key: `str | None = None` A path-like key to reference artifact in default storage, e.g., `"myfolder/myfile.fcs"`. Artifacts with the same key form a revision family.
+            revises: `Artifact | None = None` Previous version of the artifact. Triggers a revision.
+            run: `Run | None = None` The run that creates the artifact.
 
         Returns:
             A saved artifact record.
         """
+        from lamindb.core._settings import settings
+
         if not self._validated:
-            raise ValidationError("Please run `validate()` first!")
+            self.validate()
+            if not self._validated:
+                raise ValidationError("Dataset does not validate. Please curate.")
+        verbosity = settings.verbosity
+        try:
+            settings.verbosity = "warning"
+            if not self._validated:
+                # save all validated records to the current instance
+                self._update_registry_all()
 
-        self._artifact = save_artifact(
-            self._mdata,
-            description=description,
-            columns_field=self.var_index,
-            fields=self.categoricals,
-            **self._kwargs,
-            **kwargs,
-        )
+            self._artifact = save_artifact(
+                self._mdata,
+                description=description,
+                columns_field=self.var_index,
+                fields=self.categoricals,
+                key=key,
+                revises=revises,
+                run=run,
+                **self._kwargs,
+            )
+        finally:
+            settings.verbosity = verbosity
         return self._artifact
 
 
 class Curator(BaseCurator):
     """Dataset curator.
 
-    Data curation entails accurately labeling datasets with standardized metadata
-    to facilitate data integration, interpretation and analysis.
-
-    The curation flow has several steps:
-
-    1. Instantiate `Curator` from one of the following dataset objects:
+    A `Curator` object makes it easy to save validated & annotated artifacts.
 
-    - :meth:`~lamindb.Curator.from_df`
-    - :meth:`~lamindb.Curator.from_anndata`
-    - :meth:`~lamindb.Curator.from_mudata`
+    Example:
 
-    During object creation, any passed categoricals found in the object will be saved.
+    >>> curator = ln.Curator.from_df(
+    >>>     df,
+    >>>     # define validation criteria as mappings
+    >>>     columns=ln.Feature.name,  # map column names
+    >>>     categoricals={"perturbation": ln.ULabel.name},  # map categories
+    >>> )
+    >>> curator.validate()  # validate the data in df
+    >>> artifact = curate.save_artifact(description="my RNA-seq")
+    >>> artifact.describe()  # see annotations
 
-    2. Run :meth:`~lamindb.core.DataFrameCurator.validate` to check the data against the defined criteria. This method identifies:
+    `curator.validate()` maps values within `df` according to the mapping criteria and logs validated & problematic values.
 
-    - Values that can successfully validated and already exist in the registry.
-    - Values which are new and not yet validated or potentially problematic values.
+    If you find non-validated values, you have several options:
 
-    3. Determine how to handle validated and non-validated values:
-
-    - Validated values not yet in the registry can be automatically registered using :meth:`~lamindb.core.DataFrameCurator.add_validated_from`.
-    - Valid and new values can be registered using :meth:`~lamindb.core.DataFrameCurator.add_new_from`.
-    - All unvalidated values can be accessed using :meth:`~lamindb.core.DataFrameCurator.non_validated` and subsequently removed from the object at hand.
+    - new values found in the data can be registered using :meth:`~lamindb.core.DataFrameCurator.add_new_from`
+    - non-validated values can be accessed using :meth:`~lamindb.core.DataFrameCurator.non_validated` and addressed manually
     """
 
     @classmethod
@@ -1174,7 +1218,7 @@ def validate_categories(
             f"{colors.yellow(validated_hint_print)}"
         )
 
-    non_validated_hint_print = f".add_new_from('{key}')"
+    non_validated_hint_print = validated_hint_print.replace("_validated_", "_new_")
     non_validated = [i for i in non_validated if i not in values_validated]
     n_non_validated = len(non_validated)
     if n_non_validated == 0:
@@ -1239,7 +1283,9 @@ def save_artifact(
     description: str | None = None,
     organism: str | None = None,
     adata: ad.AnnData | None = None,
-    **kwargs,
+    key: str | None = None,
+    revises: Artifact | None = None,
+    run: Run | None = None,
 ) -> Artifact:
     """Save all metadata with an Artifact.
 
@@ -1249,29 +1295,43 @@ def save_artifact(
         fields: A dictionary mapping obs_column to registry_field.
         columns_field: The registry field to validate variables index against.
         organism: The organism name.
-        adata: The AnnData object to save, must be provided if data is a path.
-        kwargs: Additional keyword arguments to pass to the registry model.
+        adata: The AnnData object to save and get n_observations, must be provided if data is a path.
+        type: `Literal["dataset", "model"] | None = None` The artifact type.
+        key: `str | None = None` A path-like key to reference artifact in default storage, e.g., `"myfolder/myfile.fcs"`. Artifacts with the same key form a revision family.
+        revises: `Artifact | None = None` Previous version of the artifact. Triggers a revision.
+        run: `Run | None = None` The run that creates the artifact.
 
     Returns:
         The saved Artifact.
     """
     from ._artifact import data_is_anndata
+    from .core._data import add_labels
 
     artifact = None
     if data_is_anndata(data):
         assert adata is not None  # noqa: S101
-        artifact = Artifact.from_anndata(data, description=description, **kwargs)
+        artifact = Artifact.from_anndata(
+            data, description=description, key=key, revises=revises, run=run
+        )
         artifact.n_observations = adata.shape[0]
         data = adata
 
     elif isinstance(data, pd.DataFrame):
-        artifact = Artifact.from_df(data, description=description, **kwargs)
+        artifact = Artifact.from_df(
+            data, description=description, key=key, revises=revises, run=run
+        )
     else:
         try:
             from mudata import MuData
 
             if isinstance(data, MuData):
-                artifact = Artifact.from_mudata(data, description=description, **kwargs)
+                artifact = Artifact.from_mudata(
+                    data,
+                    description=description,
+                    key=key,
+                    revises=revises,
+                    run=run,
+                )
                 artifact.n_observations = data.n_obs
         except ImportError:
             pass
@@ -1301,7 +1361,12 @@ def save_artifact(
     else:
         raise NotImplementedError
 
-    def _add_labels(data, artifact: Artifact, fields: dict[str, FieldAttr]):
+    def _add_labels(
+        data,
+        artifact: Artifact,
+        fields: dict[str, FieldAttr],
+        feature_ref_is_name: bool | None = None,
+    ):
         features = Feature.lookup().dict()
         for key, field in fields.items():
             feature = features.get(key)
@@ -1314,16 +1379,47 @@ def save_artifact(
                 field=field,
                 **filter_kwargs_current,
             )
-            artifact.labels.add(labels, feature)
+            if len(labels) == 0:
+                continue
+            if hasattr(registry, "_name_field"):
+                label_ref_is_name = field.field.name == registry._name_field
+                add_labels(
+                    artifact,
+                    records=labels,
+                    feature=feature,
+                    feature_ref_is_name=feature_ref_is_name,
+                    label_ref_is_name=label_ref_is_name,
+                )
 
     if artifact._accessor == "MuData":
         for modality, modality_fields in fields.items():
+            column_field_modality = columns_field.get(modality)
             if modality == "obs":
-                _add_labels(data, artifact, modality_fields)
+                _add_labels(
+                    data,
+                    artifact,
+                    modality_fields,
+                    feature_ref_is_name=(
+                        None
+                        if column_field_modality is None
+                        else _ref_is_name(column_field_modality)
+                    ),
+                )
             else:
-                _add_labels(data[modality], artifact, modality_fields)
+                _add_labels(
+                    data[modality],
+                    artifact,
+                    modality_fields,
+                    feature_ref_is_name=(
+                        None
+                        if column_field_modality is None
+                        else _ref_is_name(column_field_modality)
+                    ),
+                )
     else:
-        _add_labels(data, artifact, fields)
+        _add_labels(
+            data, artifact, fields, feature_ref_is_name=_ref_is_name(columns_field)
+        )
 
     slug = ln_setup.settings.instance.slug
     if ln_setup.settings.instance.is_remote:  # pragma: no cover
@@ -1438,7 +1534,8 @@ def update_registry(
         if not validated_only:
             non_validated_records = []
             if df is not None and registry == Feature:
-                non_validated_records = Feature.from_df(df)
+                nonval_columns = Feature.inspect(df.columns, mute=True).non_validated
+                non_validated_records = Feature.from_df(df.loc[:, nonval_columns])
             else:
                 if "organism" in filter_kwargs:
                     # make sure organism record is saved to the current instance
@@ -1600,4 +1697,12 @@ def _save_organism(name: str):  # pragma: no cover
     return organism
 
 
+def _ref_is_name(field: FieldAttr) -> bool | None:
+    """Check if the reference field is a name field."""
+    from ._can_validate import get_name_field
+
+    name_field = get_name_field(field.field.model)
+    return field.field.name == name_field
+
+
 Curate = Curator  # backward compat