lamindb 0.69.0__py3-none-any.whl → 0.69.2__py3-none-any.whl

lamindb/core/storage/_backed_access.py

@@ -163,28 +163,65 @@ def read_dataframe(elem: Union[h5py.Dataset, h5py.Group]):
 
 @registry.register("h5py")
 def safer_read_partial(elem, indices):
-    if get_spec(elem).encoding_type == "":
-        if isinstance(elem, h5py.Dataset):
+    is_dataset = isinstance(elem, h5py.Dataset)
+    indices_inverse: Optional[list] = None
+    encoding_type = get_spec(elem).encoding_type
+    # h5py selection for datasets requires sorted indices
+    if is_dataset or encoding_type == "dataframe":
+        indices_increasing = []
+        indices_inverse = []
+        for indices_dim in indices:
+            if isinstance(indices_dim, np.ndarray) and not np.all(
+                np.diff(indices_dim) > 0
+            ):
+                idx_unique, idx_inverse = np.unique(indices_dim, return_inverse=True)
+                indices_increasing.append(idx_unique)
+                indices_inverse.append(idx_inverse)
+            else:
+                indices_increasing.append(indices_dim)
+                indices_inverse.append(None)
+        indices = tuple(indices_increasing)
+        if all(idx is None for idx in indices_inverse):
+            indices_inverse = None
+    result = None
+    if encoding_type == "":
+        if is_dataset:
             dims = len(elem.shape)
             if dims == 2:
-                return elem[indices]
+                result = elem[indices]
             elif dims == 1:
                 if indices[0] == slice(None):
-                    return elem[indices[1]]
+                    result = elem[indices[1]]
                 elif indices[1] == slice(None):
-                    return elem[indices[0]]
+                    result = elem[indices[0]]
         elif isinstance(elem, h5py.Group):
             try:
                 ds = CSRDataset(elem)
-                return _subset_sparse(ds, indices)
+                result = _subset_sparse(ds, indices)
             except Exception:
                 pass
-        raise ValueError(
-            "Can not get a subset of the element of type"
-            f" {type(elem).__name__} with an empty spec."
-        )
+        if result is None:
+            raise ValueError(
+                "Can not get a subset of the element of type"
+                f" {type(elem).__name__} with an empty spec."
+            )
+    else:
+        result = read_elem_partial(elem, indices=indices)
+    if indices_inverse is None:
+        return result
     else:
-        return read_elem_partial(elem, indices=indices)
+        if indices_inverse[0] is None:
+            if len(result.shape) == 2:
+                return result[:, indices_inverse[1]]
+            else:
+                return result[indices_inverse[1]]
+        elif indices_inverse[1] is None:
+            if isinstance(result, pd.DataFrame):
+                return result.iloc[indices_inverse[0]]
+            else:
+                return result[indices_inverse[0]]
+        else:
+            return result[tuple(indices_inverse)]
 
 
 @registry.register("h5py")

lamindb/core/storage/file.py

@@ -105,10 +105,7 @@ def read_adata_h5ad(filepath, **kwargs) -> ad.AnnData:
 
 
 def store_artifact(localpath: UPathStr, storagepath: UPath) -> None:
-    """Store directory or file to configured storage location.
-
-    Returns size in bytes.
-    """
+    """Store directory or file to configured storage location."""
     localpath = Path(localpath)
     if not isinstance(storagepath, LocalPathClasses):
         # this uploads files and directories
@@ -200,9 +197,7 @@ def load_to_memory(filepath: UPathStr, stream: bool = False, **kwargs):
     """
     filepath = create_path(filepath)
 
-    if filepath.suffix in (".zarr", ".zrad"):
-        stream = True
-    elif filepath.suffix != ".h5ad":
+    if filepath.suffix not in {".h5ad", ".zarr", ".zrad"}:
         stream = False
 
     if not stream:

lamindb/validation/__init__.py

@@ -0,0 +1,19 @@
+"""Validators built on LaminDB.
+
+Import the package::
+
+   from lamindb.validation import Validator, AnnDataValidator
+
+This is the complete API reference:
+
+.. autosummary::
+   :toctree: .
+
+   Validator
+   AnnDataValidator
+   Lookup
+"""
+
+from ._anndata_validator import AnnDataValidator
+from ._lookup import Lookup
+from ._validator import Validator

lamindb/validation/_anndata_validator.py

@@ -0,0 +1,117 @@
+from typing import Dict, Optional
+
+import anndata as ad
+from lnschema_core.types import FieldAttr
+from pandas.core.api import DataFrame as DataFrame
+
+import lamindb as ln
+
+from ._lookup import Lookup
+from ._register import register_artifact, register_labels
+from ._validate import validate_anndata
+from ._validator import ValidationError, Validator
+
+
+class AnnDataValidator(Validator):
+    """Lamin AnnData validator.
+
+    Args:
+        adata: The AnnData object to validate.
+        var_field: The registry field to validate variables index against.
+        obs_fields: A dictionary mapping obs_column to registry_field.
+            For example:
+            {"cell_type_ontology_id": bt.CellType.ontology_id, "donor_id": ln.ULabel.name}
+        using: The reference instance containing registries to validate against.
+    """
+
+    def __init__(
+        self,
+        adata: ad.AnnData,
+        var_field: FieldAttr,
+        obs_fields: Dict[str, FieldAttr],
+        using: str = "default",
+        verbosity: str = "hint",
+        **kwargs,
+    ) -> None:
+        self._adata = adata
+        self._var_field = var_field
+        super().__init__(
+            df=self._adata.obs,
+            fields=obs_fields,
+            using=using,
+            verbosity=verbosity,
+            **kwargs,
+        )
+        self._obs_fields = obs_fields
+        self._register_variables()
+
+    @property
+    def var_field(self) -> FieldAttr:
+        """Return the registry field to validate variables index against."""
+        return self._var_field
+
+    @property
+    def obs_fields(self) -> Dict:
+        """Return the obs fields to validate against."""
+        return self._obs_fields
+
+    def lookup(self, using: Optional[str] = None) -> Lookup:
+        """Lookup features and labels."""
+        fields = {
+            **{"feature": ln.Feature.name, "variables": self.var_field},
+            **self.obs_fields,
+        }
+        return Lookup(fields=fields, using=using or self._using)
+
+    def _register_variables(self, validated_only: bool = True, **kwargs):
+        """Register variable records."""
+        self._kwargs.update(kwargs)
+        register_labels(
+            values=self._adata.var_names,
+            field=self.var_field,
+            feature_name="variables",
+            using=self._using,
+            validated_only=validated_only,
+            kwargs=self._kwargs,
+        )
+
+    def validate(self, **kwargs) -> bool:
+        """Validate variables and categorical observations."""
+        self._kwargs.update(kwargs)
+        self._validated = validate_anndata(
+            self._adata,
+            var_field=self.var_field,
+            obs_fields=self.obs_fields,
+            **self._kwargs,
+        )
+        return self._validated
+
+    def register_labels(self, feature: str, validated_only: bool = True, **kwargs):
+        """Register labels for a feature."""
+        if feature == "variables":
+            self._register_variables(validated_only=validated_only, **kwargs)
+        else:
+            super().register_labels(feature, validated_only, **kwargs)
+
+    def register_artifact(self, description: str, **kwargs) -> ln.Artifact:
+        """Register the validated AnnData and metadata.
+
+        Args:
+            description: Description of the AnnData object.
+            **kwargs: Object level metadata.
+
+        Returns:
+            A registered artifact record.
+        """
+        self._kwargs.update(kwargs)
+        if not self._validated:
+            raise ValidationError("Please run `validate()` first!")
+
+        self._artifact = register_artifact(
+            self._adata,
+            description=description,
+            feature_field=self.var_field,
+            fields=self.obs_fields,
+            **self._kwargs,
+        )
+        return self._artifact

lamindb/validation/_lookup.py

@@ -0,0 +1,42 @@
+from typing import Dict, Optional
+
+from lamin_utils import colors, logger
+from lnschema_core.types import FieldAttr
+
+import lamindb as ln
+
+from ._validate import get_registry_instance
+
+
+class Lookup:
+    """Lookup features and labels from the reference instance."""
+
+    def __init__(
+        self, fields: Dict[str, FieldAttr], using: Optional[str] = None
+    ) -> None:
+        self._fields = fields
+        self._using = None if using == "default" else using
+        self._using_name = using or ln.setup.settings.instance.slug
+        logger.debug(f"Lookup objects from the {colors.italic(self._using_name)}")
+
+    def __getitem__(self, name):
+        if name in self._fields:
+            registry = self._fields[name].field.model
+            if self._using == "public":
+                return registry.public().lookup()
+            else:
+                return get_registry_instance(registry, self._using).lookup()
+        raise AttributeError(
+            f"'{self.__class__.__name__}' object has no attribute '{name}'"
+        )
+
+    def __repr__(self) -> str:
+        if len(self._fields) > 0:
+            fields = "\n ".join([str([key]) for key in self._fields.keys()])
+            return (
+                f"Lookup objects from the {colors.italic(self._using_name)}:\n {colors.green(fields)}\n\n"
+                "Example:\n    → categories = validator.lookup().['cell_type']\n"
+                "    → categories.alveolar_type_1_fibroblast_cell"
+            )
+        else:
+            return colors.warning("No fields are found!")

lamindb/validation/_register.py

@@ -0,0 +1,265 @@
+from typing import Dict, List, Optional, Tuple, Union
+
+import anndata as ad
+import pandas as pd
+from lamin_utils import colors, logger
+from lnschema_core.types import FieldAttr
+
+import lamindb as ln
+
+from ._validate import (
+    check_registry_organism,
+    get_registry_instance,
+    standardize_and_inspect,
+)
+
+
+def register_artifact(
+    data: Union[pd.DataFrame, ad.AnnData],
+    description: str,
+    fields: Dict[str, FieldAttr],
+    feature_field: FieldAttr,
+    **kwargs,
+) -> ln.Artifact:
+    """Register all metadata with an Artifact.
+
+    Args:
+        data: The DataFrame or AnnData object to register.
+        description: A description of the artifact.
+        fields: A dictionary mapping obs_column to registry_field.
+        feature_field: The registry field to validate variables index against.
+        kwargs: Additional keyword arguments to pass to the registry model.
+
+    Returns:
+        The registered Artifact.
+    """
+    if isinstance(data, ad.AnnData):
+        artifact = ln.Artifact.from_anndata(data, description=description)
+        artifact.n_observations = data.n_obs
+    elif isinstance(data, pd.DataFrame):
+        artifact = ln.Artifact.from_df(data, description=description)
+    else:
+        raise ValueError("data must be a DataFrame or AnnData object")
+    artifact.save()
+
+    feature_kwargs: Dict = {}
+    organism = check_registry_organism(
+        feature_field.field.model, kwargs.pop("organism", None)
+    )
+    if organism is not None:
+        feature_kwargs["organism"] = organism
+
+    if isinstance(data, ad.AnnData):
+        artifact.features.add_from_anndata(var_field=feature_field, **feature_kwargs)
+    else:
+        artifact.features.add_from_df(field=feature_field, **feature_kwargs)
+
+    features = ln.Feature.lookup().dict()
+    for feature_name, field in fields.items():
+        feature = features.get(feature_name)
+        registry = field.field.model
+        filter_kwargs = kwargs.copy()
+        organism = check_registry_organism(registry, organism)
+        if organism is not None:
+            filter_kwargs["organism"] = organism
+        df = data.obs if isinstance(data, ad.AnnData) else data
+        labels = registry.from_values(df[feature_name], field=field, **filter_kwargs)
+        artifact.labels.add(labels, feature)
+
+    slug = ln.setup.settings.instance.slug
+    logger.success(f"registered artifact in {colors.italic(slug)}")
+    if ln.setup.settings.instance.is_remote:
+        logger.info(f"🔗 https://lamin.ai/{slug}/artifact/{artifact.uid}")
+
+    return artifact
+
+
+def register_labels(
+    values: List[str],
+    field: FieldAttr,
+    feature_name: str,
+    using: Optional[str] = None,
+    validated_only: bool = True,
+    kwargs: Optional[Dict] = None,
+    df: Optional[pd.DataFrame] = None,
+) -> None:
+    """Register features or labels records in the default instance from the using instance.
+
+    Args:
+        values: A list of values to be registered as labels.
+        field: The FieldAttr object representing the field for which labels are being registered.
+        feature_name: The name of the feature to register.
+        using: The name of the instance from which to transfer labels (if applicable).
+        validated_only: If True, only register validated labels.
+        kwargs: Additional keyword arguments to pass to the registry model.
+        df: A DataFrame to register labels from.
+    """
+    filter_kwargs = {} if kwargs is None else kwargs.copy()
+    registry = field.field.model
+    if registry == ln.ULabel:
+        validated_only = False
+
+    organism = check_registry_organism(registry, filter_kwargs.pop("organism", None))
+    if organism is not None:
+        filter_kwargs["organism"] = organism
+
+    verbosity = ln.settings.verbosity
+    try:
+        ln.settings.verbosity = "error"
+        inspect_result_current = standardize_and_inspect(
+            values=values, field=field, registry=registry, **filter_kwargs
+        )
+        if not inspect_result_current.non_validated:
+            ln.settings.verbosity = verbosity
+            return
+
+        labels_registered: Dict = {"from public": [], "without reference": []}
+
+        (
+            labels_registered[f"from {using}"],
+            non_validated_labels,
+        ) = register_labels_from_using_instance(
+            inspect_result_current.non_validated,
+            field=field,
+            using=using,
+            kwargs=filter_kwargs,
+        )
+
+        public_records = (
+            registry.from_values(non_validated_labels, field=field, **filter_kwargs)
+            if non_validated_labels
+            else []
+        )
+        ln.save(public_records)
+        labels_registered["from public"] = [
+            getattr(r, field.field.name) for r in public_records
+        ]
+        labels_registered["without reference"] = [
+            i for i in non_validated_labels if i not in labels_registered["from public"]
+        ]
+
+        if not validated_only:
+            non_validated_records = []
+            if df is not None and registry == ln.Feature:
+                non_validated_records = ln.Feature.from_df(df)
+            else:
+                if "organism" in filter_kwargs:
+                    filter_kwargs["organism"] = _register_organism(name=organism)
+                for value in labels_registered["without reference"]:
+                    filter_kwargs[field.field.name] = value
+                    if registry == ln.Feature:
+                        filter_kwargs["type"] = "category"
+                    non_validated_records.append(registry(**filter_kwargs))
+            ln.save(non_validated_records)
+
+        if registry == ln.ULabel and field.field.name == "name":
+            register_ulabels_with_parent(values, field=field, feature_name=feature_name)
+    finally:
+        ln.settings.verbosity = verbosity
+
+    log_registered_labels(
+        labels_registered,
+        feature_name=feature_name,
+        model_field=f"{registry.__name__}.{field.field.name}",
+        validated_only=validated_only,
+    )
+
+
+def log_registered_labels(
+    labels_registered: Dict,
+    feature_name: str,
+    model_field: str,
+    validated_only: bool = True,
+) -> None:
+    """Log the registered labels."""
+    labels_type = "features" if feature_name == "feature" else "labels"
+    model_field = colors.italic(model_field)
+    for key, labels in labels_registered.items():
+        if not labels:
+            continue
+
+        if key == "without reference" and validated_only:
+            msg = colors.yellow(
+                f"{len(labels)} non-validated {labels_type} are not registered with {model_field}: {labels}!"
+            )
+            lookup_print = f".lookup().['{feature_name}']"
+            msg += f"\n      → to lookup categories, use {lookup_print}"
+            msg += (
+                f"\n      → to register, run {colors.yellow('register_features(validated_only=False)')}"
+                if labels_type == "features"
+                else f"\n      → to register, set {colors.yellow('validated_only=False')}"
+            )
+            logger.warning(msg)
+        else:
+            key = "" if key == "without reference" else f"{colors.green(key)} "
+            logger.success(
+                f"registered {len(labels)} {labels_type} {key}with {model_field}: {labels}"
+            )
+
+
+def register_ulabels_with_parent(
+    values: List[str], field: FieldAttr, feature_name: str
+) -> None:
+    """Register a parent label for the given labels."""
+    registry = field.field.model
+    assert registry == ln.ULabel
+    all_records = registry.from_values(values, field=field)
+    is_feature = registry.filter(name=f"is_{feature_name}").one_or_none()
+    if is_feature is None:
+        is_feature = registry(name=f"is_{feature_name}")
+        is_feature.save()
+    is_feature.children.add(*all_records)
+
+
+def register_labels_from_using_instance(
+    values: List[str],
+    field: FieldAttr,
+    using: Optional[str] = None,
+    kwargs: Optional[Dict] = None,
+) -> Tuple[List[str], List[str]]:
+    """Register features or labels records from the using instance.
+
+    Args:
+        values: A list of values to be registered as labels.
+        field: The FieldAttr object representing the field for which labels are being registered.
+        using: The name of the instance from which to transfer labels (if applicable).
+        kwargs: Additional keyword arguments to pass to the registry model.
+
+    Returns:
+        A tuple containing the list of registered labels and the list of non-registered labels.
+    """
+    kwargs = kwargs or {}
+    labels_registered = []
+    not_registered = values
+
+    if using is not None and using != "default":
+        registry = field.field.model
+        registry_using = get_registry_instance(registry, using)
+        inspect_result_using = standardize_and_inspect(
+            values=values, field=field, registry=registry_using, **kwargs
+        )
+        labels_using = registry_using.filter(
+            **{f"{field.field.name}__in": inspect_result_using.validated}
+        ).all()
+        for label_using in labels_using:
+            label_using.save()
+            labels_registered.append(getattr(label_using, field.field.name))
+        not_registered = inspect_result_using.non_validated
+
+    return labels_registered, not_registered
+
+
+def _register_organism(name: str):
+    """Register an organism record."""
+    import bionty as bt
+
+    organism = bt.Organism.filter(name=name).one_or_none()
+    if organism is None:
+        organism = bt.Organism.from_public(name=name)
+        if organism is None:
+            raise ValueError(
+                f"Organism '{name}' not found\n"
+                f"      → please register it: bt.Organism(name='{name}').save()"
+            )
+        organism.save()
+    return organism

lamindb/validation/_validate.py

@@ -0,0 +1,139 @@
+from typing import Dict, Iterable, Optional
+
+import pandas as pd
+from anndata import AnnData
+from lamin_utils import colors, logger
+from lnschema_core import Registry
+from lnschema_core.types import FieldAttr
+
+from lamindb._from_values import _print_values
+
+
+def get_registry_instance(registry: Registry, using: Optional[str] = None) -> Registry:
+    """Get a registry instance using a specific instance."""
+    if using is not None and using != "default":
+        return registry.using(using)
+    return registry
+
+
+def standardize_and_inspect(
+    values: Iterable[str], field: FieldAttr, registry: Registry, **kwargs
+):
+    """Standardize and inspect values using a registry."""
+    if hasattr(registry, "standardize"):
+        values = registry.standardize(values, field=field, mute=True, **kwargs)
+    return registry.inspect(values, field=field, mute=True, **kwargs)
+
+
+def check_registry_organism(
+    registry: Registry, organism: Optional[str] = None
+) -> Optional[str]:
+    """Check if a registry needs an organism and return the organism name."""
+    if hasattr(registry, "organism_id"):
+        import bionty as bt
+
+        if organism is None and bt.settings.organism is None:
+            raise ValueError(
+                f"{registry.__name__} registry requires an organism!\n"
+                "      → please pass an organism name via organism="
+            )
+        return organism or bt.settings.organism.name
+    return None
+
+
+def validate_categories(
+    values: Iterable[str],
+    field: FieldAttr,
+    feature_name: str,
+    using: Optional[str] = None,
+    **kwargs,
+) -> bool:
+    """Validate ontology terms in a pandas series using LaminDB registries."""
+    model_field = f"{field.field.model.__name__}.{field.field.name}"
+    logger.indent = ""
+    logger.info(
+        f"inspecting '{colors.bold(feature_name)}' by {colors.italic(model_field)}"
+    )
+    logger.indent = "   "
+
+    registry = field.field.model
+    filter_kwargs = {}
+    organism = check_registry_organism(registry, kwargs.get("organism"))
+    if organism is not None:
+        filter_kwargs["organism"] = organism
+
+    # Inspect the default instance
+    inspect_result = standardize_and_inspect(
+        values=values, field=field, registry=registry, **filter_kwargs
+    )
+    non_validated = inspect_result.non_validated
+
+    if using is not None and using != "default" and non_validated:
+        registry = get_registry_instance(registry, using)
+        # Inspect the using instance
+        inspect_result = standardize_and_inspect(
+            values=non_validated, field=field, registry=registry, **filter_kwargs
+        )
+        non_validated = inspect_result.non_validated
+
+    n_non_validated = len(non_validated)
+    if n_non_validated == 0:
+        logger.success(f"all {feature_name}s are validated")
+        return True
+    else:
+        are = "are" if n_non_validated > 1 else "is"
+        print_values = _print_values(non_validated)
+        feature_name_print = f".register_labels('{feature_name}')"
+        warning_message = (
+            f"{colors.yellow(f'{n_non_validated} terms')} {are} not validated: "
+            f"{colors.yellow(print_values)}\n      → register terms via "
+            f"{colors.yellow(feature_name_print)}"
+        )
+        logger.warning(warning_message)
+        logger.indent = ""
+        return False
+
+
+def validate_categories_in_df(
+    df: pd.DataFrame,
+    fields: Dict[str, FieldAttr],
+    using: Optional[str] = None,
+    **kwargs,
+) -> bool:
+    """Validate categories in DataFrame columns using LaminDB registries."""
+    validated = True
+    for feature_name, field in fields.items():
+        validated &= validate_categories(
+            df[feature_name],
+            field=field,
+            feature_name=feature_name,
+            using=using,
+            **kwargs,
+        )
+    return validated
+
+
+def validate_anndata(
+    adata: AnnData,
+    var_field: FieldAttr,
+    obs_fields: Dict[str, FieldAttr],
+    using: Optional[str] = None,
+    **kwargs,
+) -> bool:
+    """Inspect metadata in an AnnData object using LaminDB registries."""
+    if using is not None and using != "default":
+        logger.important(
+            f"validating metadata using registries of instance {colors.italic(using)}"
+        )
+
+    validated_var = validate_categories(
+        adata.var.index,
+        field=var_field,
+        feature_name="variables",
+        using=using,
+        **kwargs,
+    )
+    validated_obs = validate_categories_in_df(
+        adata.obs, fields=obs_fields, using=using, **kwargs
+    )
+    return validated_var and validated_obs