lamindb 0.49.3__py3-none-any.whl → 0.50.1__py3-none-any.whl

lamindb/_synonym.py

@@ -0,0 +1,203 @@
+from typing import Dict, Iterable, List, Literal, Optional, Set, Union
+
+import pandas as pd
+from django.core.exceptions import FieldDoesNotExist
+from django.db.models import QuerySet
+from lamin_utils import logger
+from lamindb_setup.dev._docs import doc_args
+from lnschema_core import Registry, SynonymsAware
+from lnschema_core.types import ListLike
+
+from lamindb.dev.utils import attach_func_to_class_method
+
+from . import _TESTING
+from ._registry import get_default_str_field
+from ._validate import _filter_query_based_on_species
+
+
+@classmethod  # type: ignore
+@doc_args(SynonymsAware.map_synonyms.__doc__)
+def map_synonyms(
+    cls,
+    synonyms: Iterable,
+    *,
+    return_mapper: bool = False,
+    case_sensitive: bool = False,
+    keep: Literal["first", "last", False] = "first",
+    synonyms_field: str = "synonyms",
+    field: Optional[str] = None,
+    **kwargs,
+) -> Union[List[str], Dict[str, str]]:
+    """{}"""
+    return _map_synonyms(
+        cls=cls,
+        synonyms=synonyms,
+        return_mapper=return_mapper,
+        case_sensitive=case_sensitive,
+        keep=keep,
+        synonyms_field=synonyms_field,
+        field=field,
+        **kwargs,
+    )
+
+
+def set_abbr(self, value: str):
+    try:
+        self.add_synonym(value, save=False)
+    except NotImplementedError:
+        pass
+    self.abbr = value
+    if not self._state.adding:
+        self.save()
+
+
+def add_synonym(
+    self,
+    synonym: Union[str, ListLike],
+    force: bool = False,
+    save: Optional[bool] = None,
+):
+    _check_synonyms_field_exist(self)
+    _add_or_remove_synonyms(
+        synonym=synonym, record=self, force=force, action="add", save=save
+    )
+
+
+def remove_synonym(self, synonym: Union[str, ListLike]):
+    _check_synonyms_field_exist(self)
+    _add_or_remove_synonyms(synonym=synonym, record=self, action="remove")
+
+
+def _add_or_remove_synonyms(
+    synonym: Union[str, Iterable],
+    record: Registry,
+    action: Literal["add", "remove"],
+    force: bool = False,
+    save: Optional[bool] = None,
+):
+    """Add or remove synonyms."""
+
+    def check_synonyms_in_all_records(synonyms: Set[str], record: Registry):
+        """Errors if input synonym is associated with other records in the DB."""
+        import pandas as pd
+        from IPython.display import display
+
+        syns_all = (
+            record.__class__.objects.exclude(synonyms="").exclude(synonyms=None).all()
+        )
+        if len(syns_all) == 0:
+            return
+        df = pd.DataFrame(syns_all.values())
+        df["synonyms"] = df["synonyms"].str.split("|")
+        df = df.explode("synonyms")
+        matches_df = df[(df["synonyms"].isin(synonyms)) & (df["id"] != record.id)]
+        if matches_df.shape[0] > 0:
+            records_df = pd.DataFrame(syns_all.filter(id__in=matches_df["id"]).values())
+            logger.error(
+                f"input synonyms {matches_df['synonyms'].unique()} already associated"
+                " with the following records:\n"
+            )
+            display(records_df)
+            raise SystemExit(AssertionError)
+
+    # passed synonyms
+    if isinstance(synonym, str):
+        syn_new_set = set([synonym])
+    else:
+        syn_new_set = set(synonym)
+    # nothing happens when passing an empty string or list
+    if len(syn_new_set) == 0:
+        return
+    # because we use | as the separator
+    if any(["|" in i for i in syn_new_set]):
+        raise AssertionError("a synonym can't contain '|'!")
+
+    # existing synonyms
+    syns_exist = record.synonyms
+    if syns_exist is None or len(syns_exist) == 0:
+        syns_exist_set = set()
+    else:
+        syns_exist_set = set(syns_exist.split("|"))
+
+    if action == "add":
+        if not force:
+            check_synonyms_in_all_records(syn_new_set, record)
+        syns_exist_set.update(syn_new_set)
+    elif action == "remove":
+        syns_exist_set = syns_exist_set.difference(syn_new_set)
+
+    if len(syns_exist_set) == 0:
+        syns_str = None
+    else:
+        syns_str = "|".join(syns_exist_set)
+
+    record.synonyms = syns_str
+
+    if save is None:
+        # if record is already in DB, save the changes to DB
+        save = not record._state.adding
+    if save:
+        record.save()
+
+
+def _check_synonyms_field_exist(record: Registry):
+    try:
+        record.__getattribute__("synonyms")
+    except AttributeError:
+        raise NotImplementedError(
+            f"No synonyms field found in table {record.__class__.__name__}!"
+        )
+
+
+def _map_synonyms(
+    cls,
+    synonyms: Iterable,
+    *,
+    return_mapper: bool = False,
+    case_sensitive: bool = False,
+    keep: Literal["first", "last", False] = "first",
+    synonyms_field: str = "synonyms",
+    field: Optional[str] = None,
+    **kwargs,
+) -> Union[List[str], Dict[str, str]]:
+    """{}"""
+    from lamin_utils._map_synonyms import map_synonyms
+
+    if isinstance(synonyms, str):
+        synonyms = [synonyms]
+    if field is None:
+        field = get_default_str_field(cls)
+    if not isinstance(field, str):
+        field = field.field.name
+
+    cls = cls.model if isinstance(cls, QuerySet) else cls
+
+    try:
+        cls._meta.get_field(synonyms_field)
+        df = _filter_query_based_on_species(orm=cls, species=kwargs.get("species"))
+    except FieldDoesNotExist:
+        df = pd.DataFrame()
+    return map_synonyms(
+        df=df,
+        identifiers=synonyms,
+        field=field,
+        return_mapper=return_mapper,
+        case_sensitive=case_sensitive,
+        keep=keep,
+        synonyms_field=synonyms_field,
+    )
+
+
+METHOD_NAMES = ["map_synonyms", "add_synonym", "remove_synonym", "set_abbr"]
+
+if _TESTING:  # type: ignore
+    from inspect import signature
+
+    SIGS = {
+        name: signature(getattr(SynonymsAware, name))
+        for name in METHOD_NAMES
+        if not name.startswith("__")
+    }
+
+for name in METHOD_NAMES:
+    attach_func_to_class_method(name, SynonymsAware, globals())

lamindb/_validate.py

@@ -0,0 +1,134 @@
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+from django.db.models import QuerySet
+from lamin_utils._inspect import InspectResult
+from lamindb_setup.dev._docs import doc_args
+from lnschema_core import Registry, ValidationAware
+from lnschema_core.types import ListLike, StrField
+
+from lamindb.dev.utils import attach_func_to_class_method
+
+from . import _TESTING
+from ._from_values import _has_species_field
+
+
+@classmethod  # type: ignore
+@doc_args(ValidationAware.inspect.__doc__)
+def inspect(
+    cls,
+    values: ListLike,
+    field: StrField,
+    *,
+    mute: bool = False,
+    **kwargs,
+) -> InspectResult:
+    """{}"""
+    return _inspect(
+        cls=cls,
+        values=values,
+        field=field,
+        mute=mute,
+        **kwargs,
+    )
+
+
+@classmethod  # type: ignore
+@doc_args(ValidationAware.validate.__doc__)
+def validate(cls, values: ListLike, field: StrField, **kwargs) -> np.ndarray[bool]:
+    """{}"""
+    return _validate(cls=cls, values=values, field=field, **kwargs)
+
+
+def _inspect(
+    cls,
+    values: ListLike,
+    field: StrField,
+    *,
+    mute: bool = False,
+    **kwargs,
+) -> Union["pd.DataFrame", Dict[str, List[str]]]:
+    """{}"""
+    from lamin_utils._inspect import inspect
+
+    if isinstance(values, str):
+        values = [values]
+    if not isinstance(field, str):
+        field = field.field.name
+
+    orm = cls.model if isinstance(cls, QuerySet) else cls
+
+    return inspect(
+        df=_filter_query_based_on_species(orm=orm, species=kwargs.get("species")),
+        identifiers=values,
+        field=str(field),
+        inspect_synonyms=True,
+        mute=mute,
+        **kwargs,
+    )
+
+
+def _validate(cls, values: ListLike, field: StrField, **kwargs) -> np.ndarray[bool]:
+    """{}"""
+    from lamin_utils._inspect import validate
+
+    if isinstance(values, str):
+        values = [values]
+    if not isinstance(field, str):
+        field = field.field.name
+
+    orm = cls.model if isinstance(cls, QuerySet) else cls
+    field_values = pd.Series(
+        _filter_query_based_on_species(
+            orm=orm, species=kwargs.get("species"), values_list_field=field
+        )
+    )
+    return validate(
+        identifiers=values, field_values=field_values, case_sensitive=True, **kwargs
+    )
+
+
+def _filter_query_based_on_species(
+    orm: Union[Registry, QuerySet],
+    species: Optional[Union[str, Registry]] = None,
+    values_list_field: Optional[str] = None,
+):
+    import pandas as pd
+
+    if values_list_field is None:
+        records = orm.all() if isinstance(orm, QuerySet) else orm.objects.all()
+    else:
+        records = orm if isinstance(orm, QuerySet) else orm.objects
+    if _has_species_field(orm):
+        # here, we can safely import lnschema_bionty
+        from lnschema_bionty._bionty import create_or_get_species_record
+
+        species_record = create_or_get_species_record(
+            species=species, orm=orm.model if isinstance(orm, QuerySet) else orm
+        )
+        if species_record is not None:
+            records = records.filter(species__name=species_record.name)
+
+    if values_list_field is None:
+        return pd.DataFrame.from_records(records.values())
+    else:
+        return records.values_list(values_list_field, flat=True)
+
+
+METHOD_NAMES = [
+    "validate",
+    "inspect",
+]
+
+if _TESTING:  # type: ignore
+    from inspect import signature
+
+    SIGS = {
+        name: signature(getattr(ValidationAware, name))
+        for name in METHOD_NAMES
+        if not name.startswith("__")
+    }
+
+for name in METHOD_NAMES:
+    attach_func_to_class_method(name, ValidationAware, globals())

lamindb/_view.py

@@ -6,17 +6,19 @@ from IPython.display import display
 from lamin_utils import colors
 from lamindb_setup import settings
 from lamindb_setup.dev._setup_schema import get_schema_module_name
-from lnschema_core import ORM
+from lnschema_core import Registry
 
 
-def view(n: int = 10, schema: Optional[str] = None, orms: Optional[List[str]] = None):
+def view(
+    n: int = 10, schema: Optional[str] = None, registries: Optional[List[str]] = None
+):
     """View data.
 
     Args:
         n: ``int = 10`` Display the last `n` rows of a table.
         schema: ``Optional[str] = None`` Schema module to view. Default's to
             `None` and displays all schema modules.
-        orms: ``Optional[List[str]] = None`` List of ORM names. Defaults to
+        registries: ``Optional[List[str]] = None`` List of Registry names. Defaults to
             `None` and lists all ORMs.
 
     Examples:
@@ -30,22 +32,26 @@ def view(n: int = 10, schema: Optional[str] = None, orms: Optional[List[str]] =
     for schema_name in schema_names:
         schema_module = importlib.import_module(get_schema_module_name(schema_name))
 
-        all_orms = {
+        all_registries = {
             orm
             for orm in schema_module.__dict__.values()
-            if inspect.isclass(orm) and issubclass(orm, ORM) and orm.__name__ != "ORM"
+            if inspect.isclass(orm)
+            and issubclass(orm, Registry)
+            and orm.__name__ != "Registry"
         }
-        if orms is not None:
-            filtered_orms = {orm for orm in all_orms if orm.__name__ in orms}
+        if registries is not None:
+            filtered_registries = {
+                orm for orm in all_registries if orm.__name__ in registries
+            }
         else:
-            filtered_orms = all_orms
+            filtered_registries = all_registries
         if len(schema_names) > 1:
             section = f"* module: {colors.green(colors.bold(schema_name))} *"
             section_no_color = f"* module: {schema_name} *"
             print("*" * len(section_no_color))
             print(section)
             print("*" * len(section_no_color))
-        for orm in sorted(filtered_orms, key=lambda x: x.__name__):
+        for orm in sorted(filtered_registries, key=lambda x: x.__name__):
             if hasattr(orm, "updated_at"):
                 df = orm.filter().order_by("-updated_at")[:n].df()
             else:

lamindb/dev/__init__.py

@@ -3,23 +3,30 @@
 .. autosummary::
    :toctree: .
 
-   ORM
+   Registry
+   Data
    QuerySet
-   Manager
+   QueryManager
    FeatureManager
+   ValidationAware
+   SynonymsAware
+   InspectResult
    datasets
    hashing
    storage
    Settings
    run_context
+   exc.ValidationError
 """
 
-from lnschema_core.models import ORM
+from lamin_utils._inspect import InspectResult
+from lnschema_core.models import Data, Registry, SynonymsAware, ValidationAware
 
-from lamindb._feature_manager import FeatureManager
-from lamindb._manager import Manager
-from lamindb._queryset import QuerySet
+from lamindb._query_manager import QueryManager
+from lamindb._query_set import QuerySet
+from lamindb.dev._feature_manager import FeatureManager
 
 from .._context import run_context
 from . import datasets  # noqa
+from . import _data, exc
 from ._settings import Settings

lamindb/dev/_data.py

@@ -0,0 +1,195 @@
+from collections import defaultdict
+from typing import Dict, List, Optional, Union
+
+from lamin_utils import logger
+from lamindb_setup.dev._docs import doc_args
+from lnschema_core.models import Data, Feature, FeatureSet, Label, Registry
+
+from .._query_set import QuerySet
+from .._registry import get_default_str_field
+from .._save import save
+from ._feature_manager import FeatureManager
+from .exc import ValidationError
+
+
+def validate_and_cast_feature(
+    feature: Union[str, Feature], records: List[Registry]
+) -> Feature:
+    if isinstance(feature, str):
+        feature_name = feature
+        feature = Feature.filter(name=feature_name).one_or_none()
+        if feature is None:
+            registries = set(
+                [record.__class__.__get_name_with_schema__() for record in records]
+            )
+            registries_str = "|".join(registries)
+            msg = (
+                f"ln.Feature(name='{feature_name}', type='category',"
+                f" registries='{registries_str}').save()"
+            )
+            raise ValidationError(f"Feature not validated. If it looks correct: {msg}")
+    return feature
+
+
+@doc_args(Data.get_labels.__doc__)
+def get_labels(
+    self,
+    feature: Optional[Union[str, Registry]] = None,
+    mute: bool = False,
+    flat_names: bool = False,
+) -> Union[QuerySet, Dict[str, QuerySet], List]:
+    """{}"""
+    if isinstance(feature, str):
+        feature_name = feature
+        feature = Feature.filter(name=feature_name).one_or_none()
+        if feature is None:
+            raise ValueError("feature doesn't exist")
+    if feature.registries is None:
+        raise ValueError("feature does not have linked labels")
+    registries_to_check = feature.registries.split("|")
+    if len(registries_to_check) > 1 and not mute:
+        logger.warning("labels come from multiple registries!")
+    qs_by_registry = {}
+    for registry in registries_to_check:
+        # currently need to distinguish between Label and non-Label, because
+        # we only have the feature information for Label
+        if registry == "core.Label":
+            links_to_labels = getattr(
+                self, self.features._accessor_by_orm[registry]
+            ).through.objects.filter(file_id=self.id, feature_id=feature.id)
+            label_ids = [link.label_id for link in links_to_labels]
+            qs_by_registry[registry] = Label.objects.filter(id__in=label_ids)
+        else:
+            qs_by_registry[registry] = getattr(
+                self, self.features._accessor_by_orm[registry]
+            ).all()
+    if flat_names:
+        # returns a flat list of names
+        from .._registry import get_default_str_field
+
+        values = []
+        for v in qs_by_registry.values():
+            values += v.list(get_default_str_field(v))
+        return values
+    if len(registries_to_check) == 1:
+        return qs_by_registry[registry]
+    else:
+        return qs_by_registry
+
+
+@doc_args(Data.add_labels.__doc__)
+def add_labels(
+    self,
+    records: Union[Registry, List[Registry], QuerySet],
+    feature: Optional[Union[str, Registry]] = None,
+) -> None:
+    """{}"""
+    if isinstance(records, (QuerySet, QuerySet.__base__)):  # need to have both
+        records = records.list()
+    if isinstance(records, str) or not isinstance(records, List):
+        records = [records]
+    if isinstance(records[0], str):  # type: ignore
+        raise ValueError(
+            "Please pass a record (a `Registry` object), not a string, e.g., via:"
+            " label"
+            f" = ln.Label(name='{records[0]}')"  # type: ignore
+        )
+    if self._state.adding:
+        raise ValueError("Please save the file or dataset before adding a label!")
+    for record in records:
+        if record._state.adding:
+            raise ValidationError(
+                f"{record} not validated. If it looks correct: record.save()"
+            )
+    feature = validate_and_cast_feature(feature, records)
+    orig_feature = feature
+    records_by_feature_orm = defaultdict(list)
+    for record in records:
+        if feature is None:
+            error_msg = "Please pass feature: add_labels(labels, feature='myfeature')"
+            record_feature = feature
+            if hasattr(record, "_feature"):
+                record_feature = record._feature
+            if record_feature is None:
+                raise ValueError(error_msg)
+            # TODO: refactor so that we don't call the following line
+            # repeatedly for the same feature
+            record_feature = validate_and_cast_feature(record_feature, [record])
+        else:
+            record_feature = feature
+        records_by_feature_orm[
+            (record_feature, record.__class__.__get_name_with_schema__())
+        ].append(record)
+    # ensure all labels are saved
+    save(records)
+    for (feature, orm_name), records in records_by_feature_orm.items():
+        getattr(self, self.features._accessor_by_orm[orm_name]).add(
+            *records, through_defaults={"feature_id": feature.id}
+        )
+    feature_set_links = self.feature_sets.through.objects.filter(file_id=self.id)
+    feature_set_ids = [link.feature_set_id for link in feature_set_links.all()]
+    # get all linked features of type Feature
+    feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
+    linked_features_by_slot = {
+        feature_set_links.filter(feature_set_id=feature_set.id)
+        .one()
+        .slot: feature_set.features.all()
+        for feature_set in feature_sets
+        if "core.Feature" == feature_set.registry
+    }
+    for (feature, orm_name), records in records_by_feature_orm.items():
+        feature = validate_and_cast_feature(feature, records)
+        msg = ""
+        if orig_feature is None:
+            records_display = ", ".join(
+                [
+                    f"'{getattr(record, get_default_str_field(record))}'"
+                    for record in records
+                ]
+            )
+            msg += f"linked labels {records_display} to feature '{feature.name}'"
+        if feature.registries is None or orm_name not in feature.registries:
+            if len(msg) > 0:
+                msg += ", "
+            msg += f"linked feature '{feature.name}' to registry '{orm_name}'"
+            if feature.registries is None:
+                feature.registries = orm_name
+            elif orm_name not in feature.registries:
+                feature.registries += f"|{orm_name}"
+            feature.save()
+        if len(msg) > 0:
+            logger.save(msg)
+        # check whether we have to update the feature set that manages labels
+        # (Feature) to account for a new feature
+        found_feature = False
+        for _, linked_features in linked_features_by_slot.items():
+            if feature in linked_features:
+                found_feature = True
+        if not found_feature:
+            if "external" not in linked_features_by_slot:
+                feature_set = FeatureSet([feature], modality="meta")
+                feature_set.save()
+                self.features.add_feature_set(feature_set, slot="external")
+                logger.save("created feature set for slot 'external'")
+            else:
+                feature_set = self.features._feature_set_by_slot["external"]
+                feature_set.features.add(feature)
+                feature_set.n += 1
+                feature_set.save()
+                logger.save(
+                    f"linked feature {feature.name} to feature set {feature_set}"
+                )
+
+
+@property  # type: ignore
+@doc_args(Data.features.__doc__)
+def features(self) -> "FeatureManager":
+    """{}"""
+    from lamindb.dev._feature_manager import FeatureManager
+
+    return FeatureManager(self)
+
+
+setattr(Data, "features", features)
+setattr(Data, "add_labels", add_labels)
+setattr(Data, "get_labels", get_labels)