lamindb 0.76.1__py3-none-any.whl → 0.76.3__py3-none-any.whl

lamindb/_curate.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import copy
-from typing import TYPE_CHECKING, Iterable, Type
+from typing import TYPE_CHECKING, Iterable
 
 import anndata as ad
 import lamindb_setup as ln_setup
@@ -84,10 +84,34 @@ class CurateLookup:
             return colors.warning("No fields are found!")
 
 
-class DataFrameCurator:
+class BaseCurator:
+    """Curate a dataset."""
+
+    def validate(self) -> bool:
+        """Validate dataset.
+
+        Returns:
+            Boolean indicating whether the dataset is validated.
+        """
+        pass
+
+    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
+        """Save the dataset as artifact.
+
+        Args:
+            description: Description of the DataFrame object.
+            **kwargs: Object level metadata.
+
+        Returns:
+            A saved artifact record.
+        """
+        pass
+
+
+class DataFrameCurator(BaseCurator):
     """Curation flow for a DataFrame object.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
 
     Args:
         df: The DataFrame object to curate.
@@ -101,10 +125,13 @@ class DataFrameCurator:
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_df(
-                df,
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name}
-            )
+        >>> curate = ln.Curator.from_df(
+        ...     df,
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     }
+        ... )
     """
 
     def __init__(
@@ -181,6 +208,7 @@ class DataFrameCurator:
             using_key=self._using_key,
             validated_only=False,
             source=self._sources.get("columns"),
+            exclude=self._exclude.get("columns"),
             **kwargs,
         )
 
@@ -196,6 +224,7 @@ class DataFrameCurator:
                 validated_only=validated_only,
                 df=self._df,  # Get the Feature type from df
                 source=self._sources.get("columns"),
+                exclude=self._exclude.get("columns"),
                 warning=False,  # Do not warn about missing columns, just an info message
                 **kwargs,
             )
@@ -247,7 +276,8 @@ class DataFrameCurator:
                 key=categorical,
                 using_key=self._using_key,
                 validated_only=validated_only,
-                sources=self._sources.get(categorical),
+                source=self._sources.get(categorical),
+                exclude=self._exclude.get(categorical),
                 **kwargs,
             )
 
@@ -260,6 +290,9 @@ class DataFrameCurator:
     def validate(self, organism: str | None = None) -> bool:
         """Validate variables and categorical observations.
 
+        Args:
+            organism: The organism name.
+
         Returns:
             Whether the DataFrame is validated.
         """
@@ -324,9 +357,11 @@ class DataFrameCurator:
 class AnnDataCurator(DataFrameCurator):
     """Curation flow for ``AnnData``.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
+
+    Note that if genes are removed from the AnnData object, the object should be recreated using :meth:`~lamindb.Curator.from_anndata`.
 
-    Note that if genes are removed from the AnnData object, the object should be recreated using :meth:`~lamindb.Curate.from_anndata`.
+    See :doc:`docs:cellxgene-curate` for instructions on how to curate against a specific cellxgene schema version.
 
     Args:
         data: The AnnData object or an AnnData-like path.
@@ -340,12 +375,15 @@ class AnnDataCurator(DataFrameCurator):
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_anndata(
-                adata,
-                var_index=bt.Gene.ensembl_gene_id,
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name},
-                organism="human",
-            )
+        >>> curate = ln.Curator.from_anndata(
+        ...     adata,
+        ...     var_index=bt.Gene.ensembl_gene_id,
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     },
+        ...     organism="human",
+        ... )
     """
 
     def __init__(
@@ -428,6 +466,7 @@ class AnnDataCurator(DataFrameCurator):
             validated_only=validated_only,
             organism=organism,
             source=self._sources.get("var_index"),
+            exclude=self._exclude.get("var_index"),
         )
 
     def _update_registry_all(self, validated_only: bool = True, **kwargs):
@@ -527,10 +566,10 @@ class AnnDataCurator(DataFrameCurator):
 class MuDataCurator:
     """Curation flow for a ``MuData`` object.
 
-    See also :class:`~lamindb.Curate`.
+    See also :class:`~lamindb.Curator`.
 
     Note that if genes or other measurements are removed from the MuData object,
-    the object should be recreated using :meth:`~lamindb.Curate.from_mudata`.
+    the object should be recreated using :meth:`~lamindb.Curator.from_mudata`.
 
     Args:
         mdata: The MuData object to curate.
@@ -547,12 +586,18 @@ class MuDataCurator:
 
     Examples:
         >>> import bionty as bt
-        >>> curate = ln.Curate.from_mudata(
-                mdata,
-                var_index={"rna": bt.Gene.ensembl_gene_id, "adt": ln.CellMarker.name},
-                categoricals={"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name},
-                organism="human",
-            )
+        >>> curate = ln.Curator.from_mudata(
+        ...     mdata,
+        ...     var_index={
+        ...         "rna": bt.Gene.ensembl_gene_id,
+        ...         "adt": ln.CellMarker.name
+        ...     },
+        ...     categoricals={
+        ...         "cell_type_ontology_id": bt.CellType.ontology_id,
+        ...         "donor_id": ln.ULabel.name
+        ...     },
+        ...     organism="human",
+        ... )
     """
 
     def __init__(
@@ -625,6 +670,8 @@ class MuDataCurator:
             using_key=self._using_key,
             validated_only=validated_only,
             dtype="number",
+            source=self._sources.get(modality, {}).get("var_index"),
+            exclude=self._exclude.get(modality, {}).get("var_index"),
             **kwargs,
         )
 
@@ -687,6 +734,8 @@ class MuDataCurator:
             using_key=self._using_key,
             validated_only=False,
             df=self._mdata[modality].obs,
+            source=self._sources.get(modality, {}).get("columns"),
+            exclude=self._exclude.get(modality, {}).get("columns"),
             **self._kwargs,  # type: ignore
             **kwargs,
         )
@@ -772,7 +821,8 @@ class MuDataCurator:
                 field=var_field,
                 key=f"{modality}_var_index",
                 using_key=self._using_key,
-                exclude=self._exclude.get(f"{modality}_var_index"),
+                source=self._sources.get(modality, {}).get("var_index"),
+                exclude=self._exclude.get(modality, {}).get("var_index"),
                 **self._kwargs,  # type: ignore
             )
             validated_var &= is_validated_var
@@ -829,19 +879,19 @@ class MuDataCurator:
         return self._artifact
 
 
-class Curate:
-    """Curation flow.
+class Curator(BaseCurator):
+    """Dataset curator.
 
     Data curation entails accurately labeling datasets with standardized metadata
     to facilitate data integration, interpretation and analysis.
 
     The curation flow has several steps:
 
-    1. Create a :class:`Curate` object corresponding to the object type that you want to curate:
+    1. Instantiate `Curator` from one of the following dataset objects:
 
-    - :meth:`~lamindb.Curate.from_df`
-    - :meth:`~lamindb.Curate.from_anndata`
-    - :meth:`~lamindb.Curate.from_mudata`
+    - :meth:`~lamindb.Curator.from_df`
+    - :meth:`~lamindb.Curator.from_anndata`
+    - :meth:`~lamindb.Curator.from_mudata`
 
     During object creation, any passed categoricals found in the object will be saved.
 
@@ -850,7 +900,7 @@ class Curate:
     - Values that can successfully validated and already exist in the registry.
     - Values which are new and not yet validated or potentially problematic values.
 
-    3. Determine how to handle validated and unvalidated values:
+    3. Determine how to handle validated and non-validated values:
 
     - Validated values not yet in the registry can be automatically registered using :meth:`~lamindb.core.DataFrameCurator.add_validated_from`.
     - Valid and new values can be registered using :meth:`~lamindb.core.DataFrameCurator.add_new_from`.
@@ -965,10 +1015,22 @@ def standardize_and_inspect(
     field: FieldAttr,
     registry: type[Record],
     standardize: bool = False,
+    exclude: str | list | None = None,
     **kwargs,
 ):
     """Standardize and inspect values using a registry."""
-    filter_kwargs = get_current_filter_kwargs(registry, kwargs)
+    # inspect exclude values in the default instance
+    values = list(values)
+    include_validated = []
+    if exclude is not None:
+        exclude = [exclude] if isinstance(exclude, str) else exclude
+        exclude = [i for i in exclude if i in values]
+        if len(exclude) > 0:
+            # exclude values are validated without source and organism
+            inspect_result_exclude = registry.inspect(exclude, field=field, mute=True)
+            # if exclude values are validated, remove them from the values
+            values = [i for i in values if i not in inspect_result_exclude.validated]
+            include_validated = inspect_result_exclude.validated
 
     if standardize:
         if hasattr(registry, "standardize") and hasattr(
@@ -976,11 +1038,17 @@ def standardize_and_inspect(
             "synonyms",  # https://github.com/laminlabs/lamindb/issues/1685
         ):
             standardized_values = registry.standardize(
-                values, field=field, mute=True, **filter_kwargs
+                values, field=field, mute=True, **kwargs
             )
             values = standardized_values
 
-    return registry.inspect(values, field=field, mute=True, **filter_kwargs)
+    inspect_result = registry.inspect(values, field=field, mute=True, **kwargs)
+    inspect_result._validated += include_validated
+    inspect_result._non_validated = [
+        i for i in inspect_result.non_validated if i not in include_validated
+    ]
+
+    return inspect_result
 
 
 def check_registry_organism(registry: Record, organism: str | None = None) -> dict:
@@ -1032,35 +1100,32 @@ def validate_categories(
         logger.indent = "   "
 
     registry = field.field.model
+
     kwargs = check_registry_organism(registry, organism)
     kwargs.update({"source": source} if source else {})
+    kwargs_current = get_current_filter_kwargs(registry, kwargs)
 
     # inspect the default instance
-    if exclude is not None:
-        exclude = [exclude] if isinstance(exclude, str) else exclude
-        # exclude values are validated without source and organism
-        inspect_result = registry.inspect(exclude, field=field, mute=True)
-        # if exclude values are validated, remove them from the values
-        values = [i for i in values if i not in inspect_result.validated]
-
     inspect_result = standardize_and_inspect(
         values=values,
         field=field,
         registry=registry,
         standardize=standardize,
-        **kwargs,
+        exclude=exclude,
+        **kwargs_current,
     )
     non_validated = inspect_result.non_validated
 
+    # inspect the using instance
     values_validated = []
     if using_key is not None and using_key != "default" and non_validated:
         registry_using = get_registry_instance(registry, using_key)
-        # inspect the using instance
         inspect_result = standardize_and_inspect(
             values=non_validated,
             field=field,
             registry=registry_using,
             standardize=standardize,
+            exclude=exclude,
             **kwargs,
         )
         non_validated = inspect_result.non_validated
@@ -1074,7 +1139,7 @@ def validate_categories(
             public_records = registry.from_values(
                 non_validated,
                 field=field,
-                **get_current_filter_kwargs(registry, kwargs),
+                **kwargs_current,
             )
             values_validated += [getattr(r, field.field.name) for r in public_records]
         finally:
@@ -1094,9 +1159,13 @@ def validate_categories(
     non_validated = [i for i in non_validated if i not in values_validated]
     n_non_validated = len(non_validated)
     if n_non_validated == 0:
-        logger.indent = ""
-        logger.success(f"{key} is validated against {colors.italic(model_field)}")
-        return True, []
+        if n_validated == 0:
+            logger.indent = ""
+            logger.success(f"{key} is validated against {colors.italic(model_field)}")
+            return True, []
+        else:
+            # validated values still need to be saved to the current instance
+            return False, []
     else:
         are = "are" if n_non_validated > 1 else "is"
         print_values = _print_values(non_validated)
@@ -1121,6 +1190,9 @@ def validate_categories_in_df(
     **kwargs,
 ) -> tuple[bool, dict]:
     """Validate categories in DataFrame columns using LaminDB registries."""
+    if not fields:
+        return True, {}
+
     if sources is None:
         sources = {}
     validated = True
@@ -1253,6 +1325,7 @@ def update_registry(
     source: Record | None = None,
     standardize: bool = True,
     warning: bool = True,
+    exclude: str | list | None = None,
     **kwargs,
 ) -> None:
     """Save features or labels records in the default instance from the using_key instance.
@@ -1312,7 +1385,8 @@ def update_registry(
             field=field,
             registry=registry,
             standardize=standardize,
-            **filter_kwargs,
+            exclude=exclude,
+            **filter_kwargs_current,
         )
         if not inspect_result_current.non_validated:
             all_labels = registry.from_values(
@@ -1331,6 +1405,7 @@ def update_registry(
             inspect_result_current.non_validated,
             field=field,
             using_key=using_key,
+            exclude=exclude,
             **filter_kwargs,
         )
 
@@ -1450,6 +1525,7 @@ def update_registry_from_using_instance(
     field: FieldAttr,
     using_key: str | None = None,
     standardize: bool = False,
+    exclude: str | list | None = None,
     **kwargs,
 ) -> tuple[list[str], list[str]]:
     """Save features or labels records from the using_key instance.
@@ -1458,6 +1534,7 @@ def update_registry_from_using_instance(
         values: A list of values to be saved as labels.
         field: The FieldAttr object representing the field for which labels are being saved.
         using_key: The name of the instance from which to transfer labels (if applicable).
+        standardize: Whether to also standardize the values.
         kwargs: Additional keyword arguments to pass to the registry model.
 
     Returns:
@@ -1474,6 +1551,7 @@ def update_registry_from_using_instance(
             field=field,
             registry=registry_using,
             standardize=standardize,
+            exclude=exclude,
             **kwargs,
         )
         labels_using = registry_using.filter(
@@ -1501,3 +1579,6 @@ def _save_organism(name: str):  # pragma: no cover
             )
         organism.save()
     return organism
+
+
+Curate = Curator  # backward compat

lamindb/_feature.py

@@ -109,18 +109,6 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
     for name, col in df.items():
         if name in categoricals:
             dtypes[name] = "cat"
-            # below is a harder feature to write, now, because it requires to
-            # query the link tables between the label Record and file or collection
-            # the original implementation fell short
-            # categorical = categoricals[name]
-            # if hasattr(
-            #     categorical, "cat"
-            # ):  # because .categories > pd2.0, .cat.categories < pd2.0
-            #     categorical = categorical.cat
-            # categories = categorical.categories
-            # categoricals_with_unmapped_categories[name] = ULabel.filter(
-            #     feature=name
-            # ).inspect(categories, "name", logging=False)["not_mapped"]
         else:
             dtypes[name] = convert_numpy_dtype_to_lamin_feature_type(col.dtype)
 
@@ -138,46 +126,9 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
         settings.verbosity = verbosity
 
     assert len(features) == len(df.columns)  # noqa: S101
-
-    # if len(categoricals_with_unmapped_categories) > 0:
-    #     n_max = 20
-    #     categoricals_with_unmapped_categories_formatted = "\n      ".join(
-    #         [
-    #             (
-    #                 f"{key} ({len(value)}): {', '.join(value)}"
-    #                 if len(value) <= 5
-    #                 else f"{key} ({len(value)}): {', '.join(value[:5])} ..."
-    #             )
-    #             for key, value in take(
-    #                 n_max, categoricals_with_unmapped_categories.items()
-    #             )
-    #         ]
-    #     )
-    #     if len(categoricals_with_unmapped_categories) > n_max:
-    #         categoricals_with_unmapped_categories_formatted += "\n      ..."
-    #     categoricals_with_unmapped_categories_formatted
-    #     logger.info(
-    #         f"{len(categoricals_with_unmapped_categories)} features have"
-    #         f" {colors.yellow('unmapped categories')}:\n     "
-    #         f" {categoricals_with_unmapped_categories_formatted}"
-    #     )
     return RecordsList(features)
 
 
-# def from_df(
-#     self,
-#     df: "pd.DataFrame",
-#     field: Optional[FieldAttr] = Feature.name,
-#     **kwargs,
-# ) -> Dict:
-#     feature_set = FeatureSet.from_df(df, field=field, **kwargs)
-#     if feature_set is not None:
-#         feature_sets = {"columns": feature_set}
-#     else:
-#         feature_sets = {}
-#     return feature_sets
-
-
 @doc_args(Feature.save.__doc__)
 def save(self, *args, **kwargs) -> Feature:
     """{}"""  # noqa: D415

lamindb/_filter.py

@@ -1,35 +1,22 @@
 from __future__ import annotations
 
-from lnschema_core import Artifact, Collection, Feature, Record
-from lnschema_core.types import VisibilityChoice
+from typing import TYPE_CHECKING
 
-from lamindb import settings
-from lamindb._query_set import QuerySet
+from lnschema_core import Artifact, Collection
 
+from ._query_set import QuerySet, process_expressions
 
-def filter(Record: type[Record], **expressions) -> QuerySet:
+if TYPE_CHECKING:
+    from lnschema_core import Record
+
+
+def filter(registry: type[Record], **expressions) -> QuerySet:
     """See :meth:`~lamindb.core.Record.filter`."""
     _using_key = None
     if "_using_key" in expressions:
         _using_key = expressions.pop("_using_key")
-    if Record in {Artifact, Collection}:
-        # visibility is set to 0 unless expressions contains id or uid equality
-        if not (
-            "id" in expressions
-            or "uid" in expressions
-            or "uid__startswith" in expressions
-        ):
-            visibility = "visibility"
-            if not any(e.startswith(visibility) for e in expressions):
-                expressions[visibility] = (
-                    VisibilityChoice.default.value
-                )  # default visibility
-            # if visibility is None, do not apply a filter
-            # otherwise, it would mean filtering for NULL values, which doesn't make
-            # sense for a non-NULLABLE column
-            elif visibility in expressions and expressions[visibility] is None:
-                expressions.pop(visibility)
-    qs = QuerySet(model=Record, using=_using_key)
+    expressions = process_expressions(registry, expressions)
+    qs = QuerySet(model=registry, using=_using_key)
     if len(expressions) > 0:
         return qs.filter(**expressions)
     else:

lamindb/_finish.py

@@ -52,7 +52,7 @@ def save_context_core(
             return None
         notebook_content = read_notebook(filepath)  # type: ignore
         is_consecutive = check_consecutiveness(
-            notebook_content, calling_statement="ln.finish()"
+            notebook_content, calling_statement=".finish()"
         )
         if not is_consecutive:
             msg = "   Do you still want to proceed with finishing? (y/n) "
@@ -148,7 +148,7 @@ def save_context_core(
             _source_code_artifact_path,
             description=f"Source of transform {transform.uid}",
             version=transform.version,
-            is_new_version_of=prev_source,
+            revises=prev_source,
             visibility=0,  # hidden file
             run=False,
         )
@@ -211,7 +211,7 @@ def save_context_core(
             report_file = ln.Artifact(
                 report_path,
                 description=f"Report of run {run.uid}",
-                is_new_version_of=prev_report,
+                revises=prev_report,
                 visibility=0,  # hidden file
                 run=False,
             )

lamindb/_from_values.py

@@ -25,9 +25,9 @@ def get_or_create_records(
     mute: bool = False,
 ) -> list[Record]:
     """Get or create records from iterables."""
-    Record = field.field.model
+    registry = field.field.model
     if create:
-        return [Record(**{field.field.name: value}) for value in iterable]
+        return [registry(**{field.field.name: value}) for value in iterable]
     creation_search_names = settings.creation.search_names
     feature: Feature = None
     organism = _get_organism_record(field, organism)
@@ -57,21 +57,23 @@ def get_or_create_records(
                     and records[0].source_id
                 ):
                     source_record = records[0].source
-            if not source_record and hasattr(Record, "public"):
+            if not source_record and hasattr(registry, "public"):
                 from bionty._bionty import get_source_record
 
-                source_record = get_source_record(Record.public(organism=organism))
+                source_record = get_source_record(
+                    registry.public(organism=organism), registry
+                )
             if source_record:
                 from bionty.core._add_ontology import check_source_in_db
 
                 check_source_in_db(
-                    registry=Record,
+                    registry=registry,
                     source=source_record,
                     update=True,
                 )
 
                 from_source = not source_record.in_db
-            elif hasattr(Record, "source_id"):
+            elif hasattr(registry, "source_id"):
                 from_source = True
             else:
                 from_source = False
@@ -97,14 +99,14 @@ def get_or_create_records(
                     logger.success(msg)
                 s = "" if len(unmapped_values) == 1 else "s"
                 print_values = colors.yellow(_print_values(unmapped_values))
-                name = Record.__name__
+                name = registry.__name__
                 n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
                 if not mute:
                     logger.warning(
                         f"{colors.red('did not create')} {name} record{s} for "
                         f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
                     )
-        if Record.__module__.startswith("bionty.") or Record == ULabel:
+        if registry.__get_schema_name__() == "bionty" or registry == ULabel:
             if isinstance(iterable, pd.Series):
                 feature = iterable.name
             feature_name = None
@@ -230,7 +232,7 @@ def create_records_from_source(
         # for custom records that are not created from public sources
         return records, iterable_idx
     # add source record to the kwargs
-    source_record = get_source_record(public_ontology)
+    source_record = get_source_record(public_ontology, model)
     kwargs.update({"source": source_record})
 
     # filter the columns in bionty df based on fields
@@ -373,6 +375,8 @@ def _get_organism_record(
     if _has_organism_field(registry) and check:
         from bionty._bionty import create_or_get_organism_record
 
-        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
+        organism_record = create_or_get_organism_record(
+            organism=organism, registry=registry
+        )
         if organism_record is not None:
             return organism_record

lamindb/_is_versioned.py

@@ -7,15 +7,13 @@ from lnschema_core.models import IsVersioned
 
 from lamindb._utils import attach_func_to_class_method
 
-from .core.versioning import get_new_path_from_uid, get_uid_from_old_version
+from .core.versioning import create_uid, get_new_path_from_uid
 
 
 # docstring handled through attach_func_to_class_method
-def _add_to_version_family(
-    self, is_new_version_of: IsVersioned, version: str | None = None
-):
+def _add_to_version_family(self, revises: IsVersioned, version: str | None = None):
     old_uid = self.uid
-    new_uid, version = get_uid_from_old_version(is_new_version_of, version)
+    new_uid, revises = create_uid(revises=revises, version=version)
     if self.__class__.__name__ == "Artifact" and self._key_is_virtual:
         old_path = self.path
         new_path = get_new_path_from_uid(

lamindb/_query_manager.py

@@ -28,7 +28,7 @@ class QueryManager(models.Manager):
         >>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))  # noqa
         >>> labels = ln.ULabel.filter(name__icontains = "label").all()
         >>> ln.ULabel(name="ULabel1").save()
-        >>> label = ln.ULabel.filter(name="ULabel1").one()
+        >>> label = ln.ULabel.get(name="ULabel1")
         >>> label.parents.set(labels)
         >>> manager = label.parents
         >>> manager.df()
@@ -57,7 +57,7 @@ class QueryManager(models.Manager):
             >>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
             >>> labels = ln.ULabel.filter(name__icontains="label").all()
             >>> ln.ULabel(name="ULabel1").save()
-            >>> label = ln.ULabel.filter(name="ULabel1").one()
+            >>> label = ln.ULabel.get(name="ULabel1")
             >>> label.parents.set(labels)
             >>> label.parents.list()
             >>> label.parents.list("name")