lamindb 0.49.3__py3-none-any.whl → 0.50.1__py3-none-any.whl

lamindb/{_orm.py → _registry.py}

@@ -1,5 +1,5 @@
 import builtins
-from typing import Dict, Iterable, List, Literal, NamedTuple, Optional, Set, Union
+from typing import Dict, Iterable, List, NamedTuple, Optional, Union
 
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
@@ -9,16 +9,17 @@ from lamin_utils import colors, logger
 from lamin_utils._lookup import Lookup
 from lamin_utils._search import search as base_search
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core import ORM
-from lnschema_core.models import format_datetime
+from lnschema_core import Registry
+from lnschema_core.models import format_field_value
 from lnschema_core.types import ListLike, StrField
 
 from lamindb.dev.utils import attach_func_to_class_method
 
 from . import _TESTING
-from ._feature_manager import create_features_df
-from ._from_values import _has_species_field, get_or_create_records
+from ._from_values import get_or_create_records
+from .dev._feature_manager import create_features_df
 from .dev._settings import settings
+from .dev._view_parents import _transform_emoji
 
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
 
@@ -27,7 +28,7 @@ class ValidationError(Exception):
     pass
 
 
-def init_self_from_db(self: ORM, existing_record: ORM):
+def init_self_from_db(self: Registry, existing_record: Registry):
     new_args = [
         getattr(existing_record, field.attname) for field in self._meta.concrete_fields
     ]
@@ -36,7 +37,7 @@ def init_self_from_db(self: ORM, existing_record: ORM):
     self._state.db = "default"
 
 
-def validate_required_fields(orm: ORM, kwargs):
+def validate_required_fields(orm: Registry, kwargs):
     required_fields = {
         k.name for k in orm._meta.fields if not k.null and k.default is None
     }
@@ -49,7 +50,7 @@ def validate_required_fields(orm: ORM, kwargs):
         raise TypeError(f"{missing_fields} are required.")
 
 
-def suggest_objects_with_same_name(orm: ORM, kwargs) -> Optional[str]:
+def suggest_objects_with_same_name(orm: Registry, kwargs) -> Optional[str]:
     if kwargs.get("name") is None:
         return None
     else:
@@ -65,9 +66,10 @@ def suggest_objects_with_same_name(orm: ORM, kwargs) -> Optional[str]:
             if results.index[0] == kwargs["name"]:
                 return "object-with-same-name-exists"
             else:
+                s = "" if results.shape[0] == 1 else "s"
+                it = "it" if results.shape[0] == 1 else "one of them"
                 msg = (
-                    "Records with similar names exist! Did you mean to load one of"
-                    " them?"
+                    f"record{s} with similar name{s} exist! did you mean to load {it}?"
                 )
                 if IPYTHON:
                     from IPython.display import display
@@ -79,7 +81,7 @@ def suggest_objects_with_same_name(orm: ORM, kwargs) -> Optional[str]:
     return None
 
 
-def __init__(orm: ORM, *args, **kwargs):
+def __init__(orm: Registry, *args, **kwargs):
     if not args:
         validate_required_fields(orm, kwargs)
         from .dev._settings import settings
@@ -97,23 +99,23 @@ def __init__(orm: ORM, *args, **kwargs):
                     existing_record = orm.filter(name=kwargs["name"]).one()
                 if existing_record is not None:
                     logger.success(
-                        f"Loaded record with exact same name{version_comment}"
+                        f"loaded record with exact same name{version_comment}"
                     )
                     init_self_from_db(orm, existing_record)
                     return None
-        super(ORM, orm).__init__(**kwargs)
+        super(Registry, orm).__init__(**kwargs)
     elif len(args) != len(orm._meta.concrete_fields):
-        raise ValueError("Please provide keyword arguments, not plain arguments")
+        raise ValueError("please provide keyword arguments, not plain arguments")
     else:
         # object is loaded from DB (**kwargs could be omitted below, I believe)
-        super(ORM, orm).__init__(*args, **kwargs)
+        super(Registry, orm).__init__(*args, **kwargs)
 
 
 def view_parents(
     self,
     field: Optional[StrField] = None,
     with_children: bool = False,
-    distance: int = 100,
+    distance: int = 5,
 ):
     from lamindb.dev._view_parents import view_parents as _view_parents
 
@@ -128,14 +130,14 @@ def view_parents(
 
 
 @classmethod  # type:ignore
-@doc_args(ORM.from_values.__doc__)
-def from_values(cls, values: ListLike, field: StrField, **kwargs) -> List["ORM"]:
+@doc_args(Registry.from_values.__doc__)
+def from_values(cls, values: ListLike, field: StrField, **kwargs) -> List["Registry"]:
     """{}"""
     if isinstance(field, str):
         field = getattr(cls, field)
     if not isinstance(field, Field):  # field is DeferredAttribute
         raise TypeError(
-            "field must be a string or an ORM field, e.g., `CellType.name`!"
+            "field must be a string or an Registry field, e.g., `CellType.name`!"
         )
     from_bionty = True if cls.__module__.startswith("lnschema_bionty.") else False
     return get_or_create_records(
@@ -144,7 +146,7 @@ def from_values(cls, values: ListLike, field: StrField, **kwargs) -> List["ORM"]
 
 
 # From: https://stackoverflow.com/a/37648265
-def _order_queryset_by_ids(queryset: QuerySet, ids: Iterable):
+def _order_query_set_by_ids(queryset: QuerySet, ids: Iterable):
     from django.db.models import Case, When
 
     preserved = Case(*[When(pk=pk, then=pos) for pos, pk in enumerate(ids)])
@@ -228,13 +230,13 @@ def _search(
         result["__ratio__"] = result.pop("__ratio__")
 
     if return_queryset:
-        return _order_queryset_by_ids(query_set, result.reset_index()["id"])
+        return _order_query_set_by_ids(query_set, result.reset_index()["id"])
     else:
         return result.fillna("")
 
 
 @classmethod  # type: ignore
-@doc_args(ORM.search.__doc__)
+@doc_args(Registry.search.__doc__)
 def search(
     cls,
     string: str,
@@ -260,7 +262,10 @@ def search(
 def _lookup(cls, field: Optional[StrField] = None) -> NamedTuple:
     """{}"""
     if field is None:
-        field = get_default_str_field(cls)
+        if cls._meta.model.__name__ == "User":
+            field = cls._meta.get_field("handle").name
+        else:
+            field = get_default_str_field(cls)
     if not isinstance(field, str):
         field = field.field.name
 
@@ -276,131 +281,12 @@ def _lookup(cls, field: Optional[StrField] = None) -> NamedTuple:
 
 
 @classmethod  # type: ignore
-@doc_args(ORM.lookup.__doc__)
+@doc_args(Registry.lookup.__doc__)
 def lookup(cls, field: Optional[StrField] = None) -> NamedTuple:
     """{}"""
     return _lookup(cls=cls, field=field)
 
 
-def _inspect(
-    cls,
-    identifiers: ListLike,
-    field: StrField,
-    *,
-    case_sensitive: bool = False,
-    inspect_synonyms: bool = True,
-    return_df: bool = False,
-    logging: bool = True,
-    **kwargs,
-) -> Union["pd.DataFrame", Dict[str, List[str]]]:
-    """{}"""
-    from lamin_utils._inspect import inspect
-
-    if not isinstance(field, str):
-        field = field.field.name
-
-    cls = cls.model if isinstance(cls, QuerySet) else cls
-
-    return inspect(
-        df=_filter_df_based_on_species(orm=cls, species=kwargs.get("species")),
-        identifiers=identifiers,
-        field=str(field),
-        case_sensitive=case_sensitive,
-        inspect_synonyms=inspect_synonyms,
-        return_df=return_df,
-        logging=logging,
-    )
-
-
-@classmethod  # type: ignore
-@doc_args(ORM.inspect.__doc__)
-def inspect(
-    cls,
-    identifiers: ListLike,
-    field: StrField,
-    *,
-    case_sensitive: bool = False,
-    inspect_synonyms: bool = True,
-    return_df: bool = False,
-    logging: bool = True,
-    **kwargs,
-) -> Union["pd.DataFrame", Dict[str, List[str]]]:
-    """{}"""
-    return _inspect(
-        cls=cls,
-        identifiers=identifiers,
-        field=field,
-        case_sensitive=case_sensitive,
-        inspect_synonyms=inspect_synonyms,
-        return_df=return_df,
-        logging=logging,
-        **kwargs,
-    )
-
-
-def _map_synonyms(
-    cls,
-    synonyms: Iterable,
-    *,
-    return_mapper: bool = False,
-    case_sensitive: bool = False,
-    keep: Literal["first", "last", False] = "first",
-    synonyms_field: str = "synonyms",
-    field: Optional[str] = None,
-    **kwargs,
-) -> Union[List[str], Dict[str, str]]:
-    """{}"""
-    from lamin_utils._map_synonyms import map_synonyms
-
-    if field is None:
-        field = get_default_str_field(cls)
-    if not isinstance(field, str):
-        field = field.field.name
-
-    cls = cls.model if isinstance(cls, QuerySet) else cls
-
-    try:
-        cls._meta.get_field(synonyms_field)
-        df = _filter_df_based_on_species(orm=cls, species=kwargs.get("species"))
-    except FieldDoesNotExist:
-        df = pd.DataFrame()
-    return map_synonyms(
-        df=df,
-        identifiers=synonyms,
-        field=field,
-        return_mapper=return_mapper,
-        case_sensitive=case_sensitive,
-        keep=keep,
-        synonyms_field=synonyms_field,
-    )
-
-
-@classmethod  # type: ignore
-@doc_args(ORM.map_synonyms.__doc__)
-def map_synonyms(
-    cls,
-    synonyms: Iterable,
-    *,
-    return_mapper: bool = False,
-    case_sensitive: bool = False,
-    keep: Literal["first", "last", False] = "first",
-    synonyms_field: str = "synonyms",
-    field: Optional[str] = None,
-    **kwargs,
-) -> Union[List[str], Dict[str, str]]:
-    """{}"""
-    return _map_synonyms(
-        cls=cls,
-        synonyms=synonyms,
-        return_mapper=return_mapper,
-        case_sensitive=case_sensitive,
-        keep=keep,
-        synonyms_field=synonyms_field,
-        field=field,
-        **kwargs,
-    )
-
-
 def describe(self):
     model_name = colors.green(self.__class__.__name__)
     msg = ""
@@ -433,7 +319,12 @@ def describe(self):
 
     # Display Provenance
     # display line by line the foreign key fields
-    emojis = {"storage": "💾", "created_by": "👤", "transform": "💫", "run": "🚗"}
+    emojis = {
+        "storage": "🗃️",
+        "created_by": "👤",
+        "transform": _transform_emoji(self.transform),
+        "run": "🚗",
+    }
     if len(foreign_key_fields) > 0:
         record_msg = f"{model_name}({''.join([f'{i}={self.__getattribute__(i)}, ' for i in direct_fields])})"  # noqa
         msg += f"{record_msg.rstrip(', )')})\n\n"
@@ -448,7 +339,7 @@ def describe(self):
         msg += related_msg
     # input of
     if self.input_of.exists():
-        values = [format_datetime(i.run_at) for i in self.input_of.all()]
+        values = [format_field_value(i.run_at) for i in self.input_of.all()]
         msg += f"⬇️ input_of ({colors.italic('core.Run')}): {values}\n    "
     msg = msg.rstrip("    ")
 
@@ -462,21 +353,19 @@ def describe(self):
     # Display Features by slot
     msg += f"{colors.green('Features')}:\n"
     # var
-    feature_sets = self.feature_sets.exclude(ref_field__startswith="core.Feature")
+    feature_sets = self.feature_sets.exclude(registry="core.Feature")
     if feature_sets.exists():
         for feature_set in feature_sets.all():
-            key_split = feature_set.ref_field.split(".")
-            if len(key_split) != 3:
+            key_split = feature_set.registry.split(".")
+            if len(key_split) == 3:
                 logger.warning(
-                    "You have a legacy entry in feature_set.field, should be format"
-                    " 'bionty.Gene.symbol'"
+                    "you have a legacy entry in feature_set.field, should be just"
+                    " 'bionty.Gene'"
                 )
             orm_name_with_schema = f"{key_split[0]}.{key_split[1]}"
-            field_name = key_split[2]
+            field_name = "id"
             related_name = feature_sets_related_models.get(orm_name_with_schema)
-            values = (
-                feature_set.__getattribute__(related_name).all()[:5].list(field_name)
-            )
+            values = feature_set.__getattribute__(related_name).all()[:5].list("id")
             slots = self.feature_sets.through.objects.filter(
                 file=self, feature_set=feature_set
             ).list("slot")
@@ -491,7 +380,7 @@ def describe(self):
 
     # obs
     # Feature, combine all features into one dataframe
-    feature_sets = self.feature_sets.filter(ref_field__startswith="core.Feature").all()
+    feature_sets = self.feature_sets.filter(registry="core.Feature").all()
     if feature_sets.exists():
         features_df = create_features_df(
             file=self, feature_sets=feature_sets.all(), exclude=True
@@ -502,7 +391,7 @@ def describe(self):
                 slot += " (metadata)"
             msg += f"  🗺️ {colors.bold(slot)}:\n"
             for _, row in df_slot.iterrows():
-                labels = self.features.get_labels(row["name"], mute=True)
+                labels = self.get_labels(row["name"], mute=True)
                 indent = ""
                 if isinstance(labels, dict):
                     msg += f"    🔗 {row['name']} ({row.registries})\n"
@@ -523,39 +412,12 @@ def describe(self):
     msg = msg.rstrip("\n")
     msg = msg.rstrip("Features:")
     verbosity = settings.verbosity
-    settings.verbosity = 2
+    settings.verbosity = 3
     logger.info(msg)
     settings.verbosity = verbosity
 
 
-def set_abbr(self, value: str):
-    try:
-        self.add_synonym(value, save=False)
-    except NotImplementedError:
-        pass
-    self.abbr = value
-    if not self._state.adding:
-        self.save()
-
-
-def _filter_df_based_on_species(
-    orm: Union[ORM, QuerySet], species: Optional[Union[str, ORM]] = None
-):
-    import pandas as pd
-
-    records = orm.all() if isinstance(orm, QuerySet) else orm.objects.all()
-    if _has_species_field(orm):
-        # here, we can safely import lnschema_bionty
-        from lnschema_bionty._bionty import create_or_get_species_record
-
-        species_record = create_or_get_species_record(species=species, orm=orm)
-        if species_record is not None:
-            records = records.filter(species__name=species_record.name)
-
-    return pd.DataFrame.from_records(records.values())
-
-
-def get_default_str_field(orm: Union[ORM, QuerySet, Manager]) -> str:
+def get_default_str_field(orm: Union[Registry, QuerySet, Manager]) -> str:
     """Get the 1st char or text field from the orm."""
     if isinstance(orm, (QuerySet, Manager)):
         orm = orm.model
@@ -583,115 +445,12 @@ def get_default_str_field(orm: Union[ORM, QuerySet, Manager]) -> str:
     return field.name
 
 
-def _add_or_remove_synonyms(
-    synonym: Union[str, Iterable],
-    record: ORM,
-    action: Literal["add", "remove"],
-    force: bool = False,
-    save: Optional[bool] = None,
-):
-    """Add or remove synonyms."""
-
-    def check_synonyms_in_all_records(synonyms: Set[str], record: ORM):
-        """Errors if input synonym is associated with other records in the DB."""
-        import pandas as pd
-        from IPython.display import display
-
-        syns_all = (
-            record.__class__.objects.exclude(synonyms="").exclude(synonyms=None).all()
-        )
-        if len(syns_all) == 0:
-            return
-        df = pd.DataFrame(syns_all.values())
-        df["synonyms"] = df["synonyms"].str.split("|")
-        df = df.explode("synonyms")
-        matches_df = df[(df["synonyms"].isin(synonyms)) & (df["id"] != record.id)]
-        if matches_df.shape[0] > 0:
-            records_df = pd.DataFrame(syns_all.filter(id__in=matches_df["id"]).values())
-            logger.error(
-                f"Input synonyms {matches_df['synonyms'].unique()} already associated"
-                " with the following records:\n(Pass `force=True` to ignore this error)"
-            )
-            display(records_df)
-            raise SystemExit(AssertionError)
-
-    # passed synonyms
-    if isinstance(synonym, str):
-        syn_new_set = set([synonym])
-    else:
-        syn_new_set = set(synonym)
-    # nothing happens when passing an empty string or list
-    if len(syn_new_set) == 0:
-        return
-    # because we use | as the separator
-    if any(["|" in i for i in syn_new_set]):
-        raise AssertionError("A synonym can't contain '|'!")
-
-    # existing synonyms
-    syns_exist = record.synonyms
-    if syns_exist is None or len(syns_exist) == 0:
-        syns_exist_set = set()
-    else:
-        syns_exist_set = set(syns_exist.split("|"))
-
-    if action == "add":
-        if not force:
-            check_synonyms_in_all_records(syn_new_set, record)
-        syns_exist_set.update(syn_new_set)
-    elif action == "remove":
-        syns_exist_set = syns_exist_set.difference(syn_new_set)
-
-    if len(syns_exist_set) == 0:
-        syns_str = None
-    else:
-        syns_str = "|".join(syns_exist_set)
-
-    record.synonyms = syns_str
-
-    if save is None:
-        # if record is already in DB, save the changes to DB
-        save = not record._state.adding
-    if save:
-        record.save()
-
-
-def _check_synonyms_field_exist(record: ORM):
-    try:
-        record.__getattribute__("synonyms")
-    except AttributeError:
-        raise NotImplementedError(
-            f"No synonyms field found in table {record.__class__.__name__}!"
-        )
-
-
-def add_synonym(
-    self,
-    synonym: Union[str, ListLike],
-    force: bool = False,
-    save: Optional[bool] = None,
-):
-    _check_synonyms_field_exist(self)
-    _add_or_remove_synonyms(
-        synonym=synonym, record=self, force=force, action="add", save=save
-    )
-
-
-def remove_synonym(self, synonym: Union[str, ListLike]):
-    _check_synonyms_field_exist(self)
-    _add_or_remove_synonyms(synonym=synonym, record=self, action="remove")
-
-
 METHOD_NAMES = [
     "__init__",
     "search",
     "lookup",
-    "map_synonyms",
-    "inspect",
-    "add_synonym",
-    "remove_synonym",
     "from_values",
     "describe",
-    "set_abbr",
     "view_parents",
 ]
 
@@ -699,13 +458,13 @@ if _TESTING:  # type: ignore
     from inspect import signature
 
     SIGS = {
-        name: signature(getattr(ORM, name))
+        name: signature(getattr(Registry, name))
         for name in METHOD_NAMES
         if not name.startswith("__")
     }
 
 for name in METHOD_NAMES:
-    attach_func_to_class_method(name, ORM, globals())
+    attach_func_to_class_method(name, Registry, globals())
 
 
 @classmethod  # type: ignore
@@ -722,7 +481,7 @@ def __get_name_with_schema__(cls) -> str:
 
 
 def select_backward(cls, **expressions):
-    logger.warning("select() is deprecated! Please rename: ORM.filter()")
+    logger.warning("select() is deprecated! please use: Registry.filter()")
     return cls.filter(**expressions)
 
 
@@ -731,6 +490,6 @@ def select(cls, **expressions):
     return select_backward(cls, **expressions)
 
 
-setattr(ORM, "__get_schema_name__", __get_schema_name__)
-setattr(ORM, "__get_name_with_schema__", __get_name_with_schema__)
-setattr(ORM, "select", select)  # backward compat
+setattr(Registry, "__get_schema_name__", __get_schema_name__)
+setattr(Registry, "__get_name_with_schema__", __get_name_with_schema__)
+setattr(Registry, "select", select)  # backward compat

lamindb/_save.py

@@ -9,7 +9,7 @@ from typing import Iterable, List, Optional, Tuple, Union, overload  # noqa
 import lamindb_setup
 from django.db import transaction
 from lamin_utils import logger
-from lnschema_core.models import ORM, File
+from lnschema_core.models import File, Registry
 
 from lamindb.dev.storage import store_object
 from lamindb.dev.storage.file import (
@@ -26,8 +26,8 @@ except ImportError:
         raise ImportError("Please install zarr: pip install zarr")
 
 
-def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
-    """Bulk save to database & storage.
+def save(records: Iterable[Registry], **kwargs) -> None:  # type: ignore
+    """Bulk save to registries & storage.
 
     Note:
 
@@ -36,10 +36,10 @@ def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
     Warning:
 
         It neither automatically creates related records nor updates existing records!
-        Use ``ORM.save()`` for these use cases.
+        Use ``Registry.save()`` for these use cases.
 
     Args:
-        records: One or multiple ``ORM`` objects.
+        records: One or multiple ``Registry`` objects.
 
     Examples:
 
@@ -63,7 +63,7 @@ def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
     """
     if isinstance(records, Iterable):
         records = set(records)
-    elif isinstance(records, ORM):
+    elif isinstance(records, Registry):
         records = {records}
 
     # we're distinguishing between files and non-files
@@ -86,11 +86,11 @@ def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
             ):
                 # save the record with parents one by one
                 logger.warning(
-                    "Now recursing through parents: "
+                    "now recursing through parents: "
                     "this only happens once, but is much slower than bulk saving"
                 )
                 logger.hint(
-                    "You can switch this off via: lb.settings.auto_save_parents = False"
+                    "you can switch this off via: lb.settings.auto_save_parents = False"
                 )
                 for record in non_files_with_parents:
                     record._save_ontology_parents()
@@ -107,7 +107,7 @@ def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
     return None
 
 
-def bulk_create(records: Iterable[ORM]):
+def bulk_create(records: Iterable[Registry]):
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
@@ -123,7 +123,7 @@ def check_and_attempt_upload(file: File) -> Optional[Exception]:
         try:
             upload_data_object(file)
         except Exception as exception:
-            logger.warning(f"Could not upload file: {file}")
+            logger.warning(f"could not upload file: {file}")
             return exception
         # copies (if ob-disk) or moves the temporary file (if in-memory) to the cache
         copy_or_move_to_cache(file)
@@ -173,7 +173,7 @@ def check_and_attempt_clearing(file: File) -> Optional[Exception]:
             if file._clear_storagekey is not None:
                 delete_storage_using_key(file, file._clear_storagekey)
                 logger.success(
-                    f"Deleted stale object at storage key {file._clear_storagekey}"
+                    f"deleted stale object at storage key {file._clear_storagekey}"
                 )
                 file._clear_storagekey = None
         except Exception as exception:
@@ -238,15 +238,16 @@ def upload_data_object(file) -> None:
     """Store and add file and its linked entries."""
     # do NOT hand-craft the storage key!
     file_storage_key = auto_storage_key_from_file(file)
+    msg = f"storing file '{file.id}' with key '{file_storage_key}'"
     if hasattr(file, "_to_store") and file._to_store and file.suffix != ".zarr":
-        logger.hint(f"storing file {file.id} with key {file_storage_key}")
+        logger.save(msg)
         store_object(file._local_filepath, file_storage_key)
     elif (
         file.suffix in {".zarr", ".zrad"}
         and hasattr(file, "_memory_rep")
         and file._memory_rep is not None
     ):
-        logger.hint(f"storing file {file.id} with key {file_storage_key}")
+        logger.save(msg)
         storagepath = lamindb_setup.settings.storage.key_to_filepath(file_storage_key)
         print_progress = partial(
             print_hook, filepath=file_storage_key, action="uploading"