lamindb 0.69.1__py3-none-any.whl → 0.69.2__py3-none-any.whl

lamindb/core/storage/file.py

@@ -105,10 +105,7 @@ def read_adata_h5ad(filepath, **kwargs) -> ad.AnnData:
 
 
 def store_artifact(localpath: UPathStr, storagepath: UPath) -> None:
-    """Store directory or file to configured storage location.
-
-    Returns size in bytes.
-    """
+    """Store directory or file to configured storage location."""
     localpath = Path(localpath)
     if not isinstance(storagepath, LocalPathClasses):
         # this uploads files and directories
@@ -200,9 +197,7 @@ def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
     """
     filepath = create_path(filepath)
 
-    if filepath.suffix in (".zarr", ".zrad"):
-        stream = True
-    elif filepath.suffix != ".h5ad":
+    if filepath.suffix not in {".h5ad", ".zarr", ".zrad"}:
         stream = False
 
     if not stream:

lamindb/validation/_anndata_validator.py

@@ -9,7 +9,7 @@ import lamindb as ln
 from ._lookup import Lookup
 from ._register import register_artifact, register_labels
 from ._validate import validate_anndata
-from ._validator import Validator
+from ._validator import ValidationError, Validator
 
 
 class AnnDataValidator(Validator):
@@ -34,6 +34,7 @@ class AnnDataValidator(Validator):
         **kwargs,
     ) -> None:
         self._adata = adata
+        self._var_field = var_field
         super().__init__(
             df=self._adata.obs,
             fields=obs_fields,
@@ -42,8 +43,7 @@ class AnnDataValidator(Validator):
             **kwargs,
         )
         self._obs_fields = obs_fields
-        self._var_field = var_field
-        self._fields = {"variables": var_field, **obs_fields}
+        self._register_variables()
 
     @property
     def var_field(self) -> FieldAttr:
@@ -65,7 +65,7 @@ class AnnDataValidator(Validator):
 
     def _register_variables(self, validated_only: bool = True, **kwargs):
         """Register variable records."""
-        self._add_kwargs(**kwargs)
+        self._kwargs.update(kwargs)
         register_labels(
             values=self._adata.var_names,
             field=self.var_field,
@@ -75,56 +75,43 @@ class AnnDataValidator(Validator):
             kwargs=self._kwargs,
         )
 
-    def register_labels(self, feature: str, validated_only: bool = True, **kwargs):
-        """Register labels for the given feature.
-
-        Args:
-            feature: The feature to register labels for.
-                if "variables", register variables.
-            validated_only: If True, only register validated labels.
-            **kwargs: Additional metadata needed.
-        """
-        if feature == "variables":
-            self._register_variables(validated_only=validated_only, **kwargs)
-        else:
-            super().register_labels(feature, validated_only, **kwargs)
-
     def validate(self, **kwargs) -> bool:
         """Validate variables and categorical observations."""
-        self._add_kwargs(**kwargs)
+        self._kwargs.update(kwargs)
         self._validated = validate_anndata(
             self._adata,
             var_field=self.var_field,
             obs_fields=self.obs_fields,
             **self._kwargs,
         )
-
         return self._validated
 
-    def register_artifact(
-        self,
-        description: str,
-        **kwargs,
-    ) -> ln.Artifact:
+    def register_labels(self, feature: str, validated_only: bool = True, **kwargs):
+        """Register labels for a feature."""
+        if feature == "variables":
+            self._register_variables(validated_only=validated_only, **kwargs)
+        else:
+            super().register_labels(feature, validated_only, **kwargs)
+
+    def register_artifact(self, description: str, **kwargs) -> ln.Artifact:
         """Register the validated AnnData and metadata.
 
         Args:
-            description: description of the AnnData object
-            **kwargs: object level metadata
+            description: Description of the AnnData object.
+            **kwargs: Object level metadata.
 
         Returns:
-            a registered artifact record
+            A registered artifact record.
         """
-        self._add_kwargs(**kwargs)
+        self._kwargs.update(kwargs)
         if not self._validated:
-            raise ValueError("please run `validate()` first!")
+            raise ValidationError("Please run `validate()` first!")
 
         self._artifact = register_artifact(
             self._adata,
             description=description,
-            var_field=self.var_field,
+            feature_field=self.var_field,
             fields=self.obs_fields,
             **self._kwargs,
         )
-
         return self._artifact

lamindb/validation/_lookup.py

@@ -5,7 +5,7 @@ from lnschema_core.types import FieldAttr
 
 import lamindb as ln
 
-from ._validate import _registry_using
+from ._validate import get_registry_instance
 
 
 class Lookup:
@@ -15,9 +15,9 @@ class Lookup:
         self, fields: Dict[str, FieldAttr], using: Optional[str] = None
     ) -> None:
         self._fields = fields
-        self._using = using
+        self._using = None if using == "default" else using
         self._using_name = using or ln.setup.settings.instance.slug
-        logger.print(f"Lookup objects from the {colors.green(self._using_name)}")
+        logger.debug(f"Lookup objects from the {colors.italic(self._using_name)}")
 
     def __getitem__(self, name):
         if name in self._fields:
@@ -25,7 +25,7 @@ class Lookup:
             if self._using == "public":
                 return registry.public().lookup()
             else:
-                return _registry_using(registry, self._using).lookup()
+                return get_registry_instance(registry, self._using).lookup()
         raise AttributeError(
             f"'{self.__class__.__name__}' object has no attribute '{name}'"
         )
@@ -33,6 +33,10 @@ class Lookup:
     def __repr__(self) -> str:
         if len(self._fields) > 0:
             fields = "\n ".join([str([key]) for key in self._fields.keys()])
-            return f"Lookup objects from the {colors.green(self._using_name)}:\n {colors.green(fields)}\n\nExample:\n    → categories = validator.lookup().cell_type\n    → categories.alveolar_type_1_fibroblast_cell"
+            return (
+                f"Lookup objects from the {colors.italic(self._using_name)}:\n {colors.green(fields)}\n\n"
+                "Example:\n    → categories = validator.lookup().['cell_type']\n"
+                "    → categories.alveolar_type_1_fibroblast_cell"
+            )
         else:
             return colors.warning("No fields are found!")

lamindb/validation/_register.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import anndata as ad
 import pandas as pd
@@ -7,24 +7,31 @@ from lnschema_core.types import FieldAttr
 
 import lamindb as ln
 
-from ._validate import _registry_using, check_if_registry_needs_organism
+from ._validate import (
+    check_registry_organism,
+    get_registry_instance,
+    standardize_and_inspect,
+)
 
 
 def register_artifact(
     data: Union[pd.DataFrame, ad.AnnData],
     description: str,
     fields: Dict[str, FieldAttr],
-    var_field: Optional[FieldAttr] = None,
+    feature_field: FieldAttr,
     **kwargs,
-):
-    """Registers all metadata with an Artifact.
+) -> ln.Artifact:
+    """Register all metadata with an Artifact.
 
     Args:
         data: The DataFrame or AnnData object to register.
         description: A description of the artifact.
         fields: A dictionary mapping obs_column to registry_field.
-        var_field: The registry field to validate variables index against.
+        feature_field: The registry field to validate variables index against.
         kwargs: Additional keyword arguments to pass to the registry model.
+
+    Returns:
+        The registered Artifact.
     """
     if isinstance(data, ad.AnnData):
         artifact = ln.Artifact.from_anndata(data, description=description)
@@ -35,30 +42,34 @@ def register_artifact(
         raise ValueError("data must be a DataFrame or AnnData object")
     artifact.save()
 
-    organism = kwargs.pop("organism", None)
+    feature_kwargs: Dict = {}
+    organism = check_registry_organism(
+        feature_field.field.model, kwargs.pop("organism", None)
+    )
+    if organism is not None:
+        feature_kwargs["organism"] = organism
 
     if isinstance(data, ad.AnnData):
-        artifact.features.add_from_anndata(var_field=var_field, organism=organism)
+        artifact.features.add_from_anndata(var_field=feature_field, **feature_kwargs)
     else:
-        artifact.features.add_from_df()
+        artifact.features.add_from_df(field=feature_field, **feature_kwargs)
 
-    # link validated obs metadata
     features = ln.Feature.lookup().dict()
     for feature_name, field in fields.items():
         feature = features.get(feature_name)
         registry = field.field.model
         filter_kwargs = kwargs.copy()
-        if check_if_registry_needs_organism(registry, organism):
+        organism = check_registry_organism(registry, organism)
+        if organism is not None:
             filter_kwargs["organism"] = organism
         df = data.obs if isinstance(data, ad.AnnData) else data
         labels = registry.from_values(df[feature_name], field=field, **filter_kwargs)
         artifact.labels.add(labels, feature)
 
-    logger.print("\n\n🎉 registered artifact in LaminDB!\n")
+    slug = ln.setup.settings.instance.slug
+    logger.success(f"registered artifact in {colors.italic(slug)}")
     if ln.setup.settings.instance.is_remote:
-        logger.print(
-            f"🔗 https://lamin.ai/{ln.setup.settings.instance.slug}/artifact/{artifact.uid}"
-        )
+        logger.info(f"🔗 https://lamin.ai/{slug}/artifact/{artifact.uid}")
 
     return artifact
 
@@ -69,8 +80,9 @@ def register_labels(
     feature_name: str,
     using: Optional[str] = None,
     validated_only: bool = True,
-    kwargs: Dict = None,
-):
+    kwargs: Optional[Dict] = None,
+    df: Optional[pd.DataFrame] = None,
+) -> None:
     """Register features or labels records in the default instance from the using instance.
 
     Args:
@@ -80,28 +92,29 @@ def register_labels(
         using: The name of the instance from which to transfer labels (if applicable).
         validated_only: If True, only register validated labels.
         kwargs: Additional keyword arguments to pass to the registry model.
+        df: A DataFrame to register labels from.
     """
-    if kwargs is None:
-        kwargs = {}
+    filter_kwargs = {} if kwargs is None else kwargs.copy()
     registry = field.field.model
+    if registry == ln.ULabel:
+        validated_only = False
+
+    organism = check_registry_organism(registry, filter_kwargs.pop("organism", None))
+    if organism is not None:
+        filter_kwargs["organism"] = organism
 
-    check_if_registry_needs_organism(registry, kwargs.get("organism"))
     verbosity = ln.settings.verbosity
     try:
         ln.settings.verbosity = "error"
-        # for labels that are registered in the using instance, transfer them to the current instance
-        # first inspect the current instance
-        inspect_result_current = registry.inspect(
-            values, field=field, mute=True, **kwargs
+        inspect_result_current = standardize_and_inspect(
+            values=values, field=field, registry=registry, **filter_kwargs
         )
-        if len(inspect_result_current.non_validated) == 0:
-            # everything is validated in the current instance, no need to register
+        if not inspect_result_current.non_validated:
             ln.settings.verbosity = verbosity
             return
 
         labels_registered: Dict = {"from public": [], "without reference": []}
 
-        # register labels from the using instance
         (
             labels_registered[f"from {using}"],
             non_validated_labels,
@@ -109,71 +122,92 @@ def register_labels(
             inspect_result_current.non_validated,
             field=field,
             using=using,
-            kwargs=kwargs,
+            kwargs=filter_kwargs,
         )
 
-        # for labels that are not registered in the using instance, register them in the current instance
-        from_values_records = (
-            registry.from_values(non_validated_labels, field=field, **kwargs)
-            if len(non_validated_labels) > 0
+        public_records = (
+            registry.from_values(non_validated_labels, field=field, **filter_kwargs)
+            if non_validated_labels
             else []
         )
-        ln.save(from_values_records)
+        ln.save(public_records)
         labels_registered["from public"] = [
-            getattr(r, field.field.name) for r in from_values_records
+            getattr(r, field.field.name) for r in public_records
         ]
         labels_registered["without reference"] = [
             i for i in non_validated_labels if i not in labels_registered["from public"]
         ]
+
         if not validated_only:
             non_validated_records = []
-            for value in labels_registered["without reference"]:
-                kwargs[field.field.name] = value
-                if registry.__name__ == "Feature":
-                    kwargs["type"] = "category"
-                # register non-validated labels
-                non_validated_records.append(registry(**kwargs))
+            if df is not None and registry == ln.Feature:
+                non_validated_records = ln.Feature.from_df(df)
+            else:
+                if "organism" in filter_kwargs:
+                    filter_kwargs["organism"] = _register_organism(name=organism)
+                for value in labels_registered["without reference"]:
+                    filter_kwargs[field.field.name] = value
+                    if registry == ln.Feature:
+                        filter_kwargs["type"] = "category"
+                    non_validated_records.append(registry(**filter_kwargs))
             ln.save(non_validated_records)
 
-        # for ulabels, also register a parent label: is_{feature_name}
         if registry == ln.ULabel and field.field.name == "name":
-            register_ulabels_with_parent(values, field)
+            register_ulabels_with_parent(values, field=field, feature_name=feature_name)
     finally:
         ln.settings.verbosity = verbosity
+
     log_registered_labels(
-        labels_registered, feature_name=feature_name, validated_only=validated_only
+        labels_registered,
+        feature_name=feature_name,
+        model_field=f"{registry.__name__}.{field.field.name}",
+        validated_only=validated_only,
     )
 
 
 def log_registered_labels(
-    labels_registered: Dict, feature_name: str, validated_only: bool = True
-):
+    labels_registered: Dict,
+    feature_name: str,
+    model_field: str,
+    validated_only: bool = True,
+) -> None:
     """Log the registered labels."""
+    labels_type = "features" if feature_name == "feature" else "labels"
+    model_field = colors.italic(model_field)
     for key, labels in labels_registered.items():
-        if len(labels) > 0:
-            if key == "without reference" and validated_only:
-                msg = (
-                    f"{len(labels)} non-validated labels are not registered: {labels}!\n"
-                    "      → to lookup categories, use `.lookup().{feature_name}`\n"
-                    "      → to register, set `validated_only=False`"
-                )
-                logger.warning(colors.yellow(msg))
-                continue
+        if not labels:
+            continue
+
+        if key == "without reference" and validated_only:
+            msg = colors.yellow(
+                f"{len(labels)} non-validated {labels_type} are not registered with {model_field}: {labels}!"
+            )
+            lookup_print = f".lookup().['{feature_name}']"
+            msg += f"\n      → to lookup categories, use {lookup_print}"
+            msg += (
+                f"\n      → to register, run {colors.yellow('register_features(validated_only=False)')}"
+                if labels_type == "features"
+                else f"\n      → to register, set {colors.yellow('validated_only=False')}"
+            )
+            logger.warning(msg)
+        else:
+            key = "" if key == "without reference" else f"{colors.green(key)} "
             logger.success(
-                f"registered {len(labels)} records {colors.green(key)}: {labels}"
+                f"registered {len(labels)} {labels_type} {key}with {model_field}: {labels}"
             )
 
 
-def register_ulabels_with_parent(values: List[str], field: FieldAttr):
+def register_ulabels_with_parent(
+    values: List[str], field: FieldAttr, feature_name: str
+) -> None:
     """Register a parent label for the given labels."""
     registry = field.field.model
     assert registry == ln.ULabel
     all_records = registry.from_values(values, field=field)
-    is_feature = registry.filter(name=f"is_{field.field.name}").one_or_none()
+    is_feature = registry.filter(name=f"is_{feature_name}").one_or_none()
     if is_feature is None:
-        is_feature = registry(name=f"is_{field.field.name}")
+        is_feature = registry(name=f"is_{feature_name}")
         is_feature.save()
-    # link all labels to the parent label
     is_feature.children.add(*all_records)
 
 
@@ -181,8 +215,8 @@ def register_labels_from_using_instance(
     values: List[str],
     field: FieldAttr,
     using: Optional[str] = None,
-    kwargs: Dict = None,
-):
+    kwargs: Optional[Dict] = None,
+) -> Tuple[List[str], List[str]]:
     """Register features or labels records from the using instance.
 
     Args:
@@ -190,20 +224,20 @@ def register_labels_from_using_instance(
         field: The FieldAttr object representing the field for which labels are being registered.
         using: The name of the instance from which to transfer labels (if applicable).
         kwargs: Additional keyword arguments to pass to the registry model.
+
+    Returns:
+        A tuple containing the list of registered labels and the list of non-registered labels.
     """
-    if kwargs is None:
-        kwargs = {}
+    kwargs = kwargs or {}
     labels_registered = []
     not_registered = values
+
     if using is not None and using != "default":
         registry = field.field.model
-        registry_using = _registry_using(registry, using)
-        # then inspect the using instance
-        inspect_result_using = registry_using.inspect(
-            values, field=field, mute=True, **kwargs
+        registry_using = get_registry_instance(registry, using)
+        inspect_result_using = standardize_and_inspect(
+            values=values, field=field, registry=registry_using, **kwargs
         )
-        # register the labels that are validated in the using instance
-        # TODO: filter kwargs
         labels_using = registry_using.filter(
             **{f"{field.field.name}__in": inspect_result_using.validated}
         ).all()
@@ -211,4 +245,21 @@ def register_labels_from_using_instance(
             label_using.save()
             labels_registered.append(getattr(label_using, field.field.name))
         not_registered = inspect_result_using.non_validated
+
     return labels_registered, not_registered
+
+
+def _register_organism(name: str):
+    """Register an organism record."""
+    import bionty as bt
+
+    organism = bt.Organism.filter(name=name).one_or_none()
+    if organism is None:
+        organism = bt.Organism.from_public(name=name)
+        if organism is None:
+            raise ValueError(
+                f"Organism '{name}' not found\n"
+                f"      → please register it: bt.Organism(name='{name}').save()"
+            )
+        organism.save()
+    return organism

lamindb/validation/_validate.py

@@ -9,27 +9,36 @@ from lnschema_core.types import FieldAttr
 from lamindb._from_values import _print_values
 
 
-def _registry_using(registry: Registry, using: Optional[str] = None) -> Registry:
+def get_registry_instance(registry: Registry, using: Optional[str] = None) -> Registry:
     """Get a registry instance using a specific instance."""
-    return (
-        registry.using(using) if using is not None and using != "default" else registry
-    )
+    if using is not None and using != "default":
+        return registry.using(using)
+    return registry
 
 
-def check_if_registry_needs_organism(
-    registry: Registry, organism: Optional[str] = None
+def standardize_and_inspect(
+    values: Iterable[str], field: FieldAttr, registry: Registry, **kwargs
 ):
-    """Check if a registry needs an organism."""
+    """Standardize and inspect values using a registry."""
+    if hasattr(registry, "standardize"):
+        values = registry.standardize(values, field=field, mute=True, **kwargs)
+    return registry.inspect(values, field=field, mute=True, **kwargs)
+
+
+def check_registry_organism(
+    registry: Registry, organism: Optional[str] = None
+) -> Optional[str]:
+    """Check if a registry needs an organism and return the organism name."""
     if hasattr(registry, "organism_id"):
-        if organism is None:
+        import bionty as bt
+
+        if organism is None and bt.settings.organism is None:
             raise ValueError(
                 f"{registry.__name__} registry requires an organism!\n"
                 "      → please pass an organism name via organism="
             )
-        else:
-            return True
-    else:
-        return False
+        return organism or bt.settings.organism.name
+    return None
 
 
 def validate_categories(
@@ -38,7 +47,7 @@ def validate_categories(
     feature_name: str,
     using: Optional[str] = None,
     **kwargs,
-):
+) -> bool:
     """Validate ontology terms in a pandas series using LaminDB registries."""
     model_field = f"{field.field.model.__name__}.{field.field.name}"
     logger.indent = ""
@@ -48,40 +57,41 @@ def validate_categories(
     logger.indent = "   "
 
     registry = field.field.model
-    filter_kwargs = {}  # type: Dict[str, str]
-    organism = kwargs.get("organism")
-    if check_if_registry_needs_organism(registry, organism):
+    filter_kwargs = {}
+    organism = check_registry_organism(registry, kwargs.get("organism"))
+    if organism is not None:
         filter_kwargs["organism"] = organism
-    # inspect the default instance
-    inspect_result = registry.inspect(values, field=field, mute=True, **filter_kwargs)
+
+    # Inspect the default instance
+    inspect_result = standardize_and_inspect(
+        values=values, field=field, registry=registry, **filter_kwargs
+    )
     non_validated = inspect_result.non_validated
-    if using is not None and using != "default" and len(non_validated) > 0:
-        registry = _registry_using(registry, using)
-        # inspect the using instance
-        inspect_result = registry.inspect(
-            non_validated, field=field, mute=True, **filter_kwargs
+
+    if using is not None and using != "default" and non_validated:
+        registry = get_registry_instance(registry, using)
+        # Inspect the using instance
+        inspect_result = standardize_and_inspect(
+            values=non_validated, field=field, registry=registry, **filter_kwargs
         )
         non_validated = inspect_result.non_validated
 
-    # if all terms are validated
     n_non_validated = len(non_validated)
     if n_non_validated == 0:
-        validated = True
         logger.success(f"all {feature_name}s are validated")
+        return True
     else:
         are = "are" if n_non_validated > 1 else "is"
         print_values = _print_values(non_validated)
-        feature_name_print = f"`.register_labels('{feature_name}')`"
+        feature_name_print = f".register_labels('{feature_name}')"
         warning_message = (
             f"{colors.yellow(f'{n_non_validated} terms')} {are} not validated: "
             f"{colors.yellow(print_values)}\n      → register terms via "
-            f"{colors.red(feature_name_print)}"
+            f"{colors.yellow(feature_name_print)}"
         )
         logger.warning(warning_message)
-        validated = False
-    logger.indent = ""
-
-    return validated
+        logger.indent = ""
+        return False
 
 
 def validate_categories_in_df(
@@ -89,9 +99,8 @@ def validate_categories_in_df(
     fields: Dict[str, FieldAttr],
     using: Optional[str] = None,
     **kwargs,
-):
+) -> bool:
     """Validate categories in DataFrame columns using LaminDB registries."""
-    # start validation
     validated = True
     for feature_name, field in fields.items():
         validated &= validate_categories(
@@ -113,7 +122,9 @@ def validate_anndata(
 ) -> bool:
     """Inspect metadata in an AnnData object using LaminDB registries."""
     if using is not None and using != "default":
-        logger.important(f"validating metadata using registries of instance `{using}`")
+        logger.important(
+            f"validating metadata using registries of instance {colors.italic(using)}"
+        )
 
     validated_var = validate_categories(
         adata.var.index,
@@ -123,9 +134,6 @@ def validate_anndata(
         **kwargs,
     )
     validated_obs = validate_categories_in_df(
-        adata.obs,
-        fields=obs_fields,
-        using=using,
-        **kwargs,
+        adata.obs, fields=obs_fields, using=using, **kwargs
     )
-    return validated_var & validated_obs
+    return validated_var and validated_obs