lamindb 0.71.2__py3-none-any.whl → 0.71.3__py3-none-any.whl

lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.71.2"
+__version__ = "0.71.3"
 
 import os as _os
 

lamindb/_artifact.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import shutil
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, Mapping
 
 import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
+import psutil
 from anndata import AnnData
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
@@ -204,15 +206,23 @@ def get_stat_or_artifact(
             return size, hash, hash_type, n_objects
     else:
         if path.is_dir():
-            md5s = []
-            size = 0
-            for subpath in path.rglob("*"):
-                if not subpath.is_file():
-                    continue
-                size += subpath.stat().st_size
-                md5s.append(hash_file(subpath)[0])
-            hash, hash_type = hash_md5s_from_dir(md5s)
-            n_objects = len(md5s)
+            files = (subpath for subpath in path.rglob("*") if subpath.is_file())
+
+            def hash_size(file):
+                file_size = file.stat().st_size
+                return hash_file(file, file_size)[0], file_size
+
+            n_workers = len(psutil.Process().cpu_affinity())
+            if n_workers > 1:
+                with ThreadPoolExecutor(n_workers) as pool:
+                    hashes_sizes = pool.map(hash_size, files)
+            else:
+                hashes_sizes = map(hash_size, files)
+            hashes, sizes = zip(*hashes_sizes)
+
+            hash, hash_type = hash_md5s_from_dir(hashes)
+            n_objects = len(hashes)
+            size = sum(sizes)
         else:
             hash, hash_type = hash_file(path)
             size = stat.st_size

lamindb/_can_validate.py

@@ -80,7 +80,9 @@ def _inspect(
 
     # inspect in the DB
     result_db = inspect(
-        df=_filter_query_based_on_organism(queryset=queryset, organism=organism),
+        df=_filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        ),
         identifiers=values,
         field=field,
         mute=mute,
@@ -161,6 +163,7 @@ def _validate(
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
+            field=field,
             organism=organism,
             values_list_field=field,
         ),
@@ -284,7 +287,9 @@ def _standardize(
 
     try:
         orm._meta.get_field(synonyms_field)
-        df = _filter_query_based_on_organism(queryset=queryset, organism=organism)
+        df = _filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        )
     except FieldDoesNotExist:
         df = pd.DataFrame()
 
@@ -439,6 +444,7 @@ def _check_synonyms_field_exist(record: Registry):
 
 def _filter_query_based_on_organism(
     queryset: QuerySet,
+    field: str,
     organism: str | Registry | None = None,
     values_list_field: str | None = None,
 ):
@@ -447,7 +453,7 @@ def _filter_query_based_on_organism(
 
     orm = queryset.model
 
-    if _has_organism_field(orm):
+    if _has_organism_field(orm) and not field.endswith("id"):
         # here, we can safely import lnschema_bionty
         from lnschema_bionty._bionty import create_or_get_organism_record
 

lamindb/_collection.py

@@ -103,9 +103,9 @@ def __init__(
             if meta._state.adding:
                 raise ValueError("Save meta artifact before creating collection!")
             if not feature_sets:
-                feature_sets = meta.features._feature_set_by_slot
+                feature_sets = meta.features.feature_set_by_slot
             else:
-                if len(meta.features._feature_set_by_slot) > 0:
+                if len(meta.features.feature_set_by_slot) > 0:
                     logger.info("overwriting feature sets linked to artifact")
     # we ignore collections in trash containing the same hash
     if hash is not None:
@@ -129,7 +129,7 @@ def __init__(
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         update_attributes(collection, {"description": description, "name": name})
-        for slot, feature_set in collection.features._feature_set_by_slot.items():
+        for slot, feature_set in collection.features.feature_set_by_slot.items():
             if slot in feature_sets:
                 if not feature_sets[slot] == feature_set:
                     collection.feature_sets.remove(feature_set)

lamindb/_from_values.py

@@ -102,6 +102,9 @@ def get_existing_records(
 
     # standardize based on the DB reference
     # log synonyms mapped terms
+    print("field", field)
+    print("organism", kwargs.get("organism"))
+    print("public_source", kwargs.get("public_source"))
     result = model.inspect(
         iterable_idx,
         field=field,
@@ -185,8 +188,15 @@ def create_records_from_public(
 
     # create the corresponding bionty object from model
     try:
+        # TODO: more generic
+        organism = kwargs.get("organism")
+        if field.field.name == "ensembl_gene_id":
+            if iterable_idx[0].startswith("ENSG"):
+                organism = "human"
+            elif iterable_idx[0].startswith("ENSMUSG"):
+                organism = "mouse"
         public_ontology = model.public(
-            organism=kwargs.get("organism"), public_source=kwargs.get("public_source")
+            organism=organism, public_source=kwargs.get("public_source")
         )
     except Exception:
         # for custom records that are not created from public sources
@@ -223,8 +233,15 @@ def create_records_from_public(
         bionty_kwargs, multi_msg = _bulk_create_dicts_from_df(
             keys=mapped_values, column_name=field.field.name, df=bionty_df
         )
+        organism_kwargs = {}
+        if "organism" not in kwargs:
+            organism_record = _get_organism_record(
+                field, public_ontology.organism, force=True
+            )
+            if organism_record is not None:
+                organism_kwargs["organism"] = organism_record
         for bk in bionty_kwargs:
-            records.append(model(**bk, **kwargs))
+            records.append(model(**bk, **kwargs, **organism_kwargs))
 
         # number of records that matches field (not synonyms)
         validated = result.validated
@@ -260,10 +277,11 @@ def index_iterable(iterable: Iterable) -> pd.Index:
     return idx[(idx != "") & (~idx.isnull())]
 
 
-def _print_values(names: list, n: int = 20) -> str:
-    names = list(set(names))
-    print_values = ", ".join([f"'{name}'" for name in names[:n] if name != "None"])
-    if len(names) > n:
+def _print_values(names: Iterable, n: int = 20) -> str:
+    names = (name for name in names if name != "None")
+    unique_names = list(dict.fromkeys(names))[:n]
+    print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if len(unique_names) > n:
         print_values += ", ..."
     return print_values
 
@@ -334,9 +352,13 @@ def _has_organism_field(orm: Registry) -> bool:
         return False
 
 
-def _get_organism_record(field: StrField, organism: str | Registry) -> Registry:
+def _get_organism_record(
+    field: StrField, organism: str | Registry, force: bool = False
+) -> Registry:
     model = field.field.model
-    if _has_organism_field(model):
+    check = True if force else field.field.name != "ensembl_gene_id"
+
+    if _has_organism_field(model) and check:
         from lnschema_bionty._bionty import create_or_get_organism_record
 
         organism_record = create_or_get_organism_record(organism=organism, orm=model)

lamindb/_registry.py

@@ -2,14 +2,12 @@ from __future__ import annotations
 
 import builtins
 from typing import TYPE_CHECKING, Iterable, List, NamedTuple
-from uuid import UUID
 
 import dj_database_url
 import lamindb_setup as ln_setup
-import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from django.db import connections
-from django.db.models import Manager, QuerySet
+from django.db.models import Manager, Q, QuerySet
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
 from lamin_utils._search import search as base_search
@@ -26,6 +24,7 @@ from lamindb.core._settings import settings
 from ._from_values import get_or_create_records
 
 if TYPE_CHECKING:
+    import pandas as pd
     from lnschema_core.types import ListLike, StrField
 
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
@@ -61,20 +60,15 @@ def suggest_objects_with_same_name(orm: Registry, kwargs) -> str | None:
     if kwargs.get("name") is None:
         return None
     else:
-        results = orm.search(kwargs["name"])
-        if results.shape[0] == 0:
+        queryset = orm.search(kwargs["name"])
+        if not queryset.exists():  # empty queryset
             return None
-
-        # subset results to those with at least 0.90 levensteihn distance
-        results = results.loc[results.score >= 90]
-
-        # test for exact match
-        if len(results) > 0:
-            if results.index[0] == kwargs["name"]:
-                return "object-with-same-name-exists"
+        else:
+            for record in queryset:
+                if record.name == kwargs["name"]:
+                    return "object-with-same-name-exists"
             else:
-                s = "" if results.shape[0] == 1 else "s"
-                it = "it" if results.shape[0] == 1 else "one of them"
+                s, it = ("", "it") if len(queryset) == 1 else ("s", "one of them")
                 msg = (
                     f"record{s} with similar name{s} exist! did you mean to load {it}?"
                 )
@@ -83,9 +77,9 @@ def suggest_objects_with_same_name(orm: Registry, kwargs) -> str | None:
 
                     logger.warning(f"{msg}")
                     if settings._verbosity_int >= 1:
-                        display(results)
+                        display(queryset.df())
                 else:
-                    logger.warning(f"{msg}\n{results}")
+                    logger.warning(f"{msg}\n{queryset}")
     return None
 
 
@@ -162,80 +156,42 @@ def _search(
     string: str,
     *,
     field: StrField | list[StrField] | None = None,
-    limit: int | None = 10,
-    return_queryset: bool = False,
+    limit: int | None = 20,
     case_sensitive: bool = False,
-    synonyms_field: StrField | None = "synonyms",
     using_key: str | None = None,
-) -> pd.DataFrame | QuerySet:
-    queryset = _queryset(cls, using_key=using_key)
-    orm = queryset.model
-
-    def _search_single_field(
-        string: str,
-        field: StrField | None,
-        synonyms_field: StrField | None = "synonyms",
-    ) -> pd.DataFrame:
-        field = get_default_str_field(orm=orm, field=field)
-
-        try:
-            orm._meta.get_field(synonyms_field)
-            synonyms_field_exists = True
-        except FieldDoesNotExist:
-            synonyms_field_exists = False
-
-        if synonyms_field is not None and synonyms_field_exists:
-            df = pd.DataFrame(queryset.values("uid", field, synonyms_field))
-        else:
-            df = pd.DataFrame(queryset.values("uid", field))
-
-        return base_search(
-            df=df,
-            string=string,
-            field=field,
-            limit=limit,
-            synonyms_field=str(synonyms_field),
-            case_sensitive=case_sensitive,
-        )
-
-    # search in both key and description fields for Artifact
-    if orm._meta.model.__name__ == "Artifact" and field is None:
-        field = ["key", "description"]
-
-    if not isinstance(field, List):
-        field = [field]
-
-    results = []
-    for fd in field:
-        result_field = _search_single_field(
-            string=string, field=fd, synonyms_field=synonyms_field
-        )
-        results.append(result_field)
-        # turn off synonyms search after the 1st field
-        synonyms_field = None
-
-    if len(results) > 1:
-        result = (
-            pd.concat([r.reset_index() for r in results], join="outer")
-            .drop(columns=["index"], errors="ignore")
-            .set_index("uid")
-        )
-    else:
-        result = results[0]
-
-    # remove results that have __ratio__ 0
-    if "__ratio__" in result.columns:
-        result = result[result["__ratio__"] > 0].sort_values(
-            "__ratio__", ascending=False
-        )
-        # restrict to 1 decimal
-        # move the score to be the last column
-        result["score"] = result.pop("__ratio__").round(1)
-
-    if return_queryset:
-        return _order_queryset_by_ids(queryset, result.reset_index()["uid"])
+) -> QuerySet:
+    input_queryset = _queryset(cls, using_key=using_key)
+    orm = input_queryset.model
+    if field is None:
+        fields = [
+            field.name
+            for field in orm._meta.fields
+            if field.get_internal_type() in {"CharField", "TextField"}
+        ]
     else:
-        return result.fillna("")
+        if not isinstance(field, list):
+            fields_input = [field]
+        else:
+            fields_input = field
+        fields = []
+        for field in fields_input:
+            if not isinstance(field, str):
+                try:
+                    fields.append(field.field.name)
+                except AttributeError as error:
+                    raise TypeError(
+                        "Please pass a Registry string field, e.g., `CellType.name`!"
+                    ) from error
+            else:
+                fields.append(field)
+    expression = Q()
+    case_sensitive_i = "" if case_sensitive else "i"
+    for field in fields:
+        # Construct the keyword for the Q object dynamically
+        query = {f"{field}__{case_sensitive_i}contains": string}
+        expression |= Q(**query)  # Unpack the dictionary into Q()
+    output_queryset = input_queryset.filter(expression)[:limit]
+    return output_queryset
 
 
 @classmethod  # type: ignore
@@ -246,19 +202,15 @@ def search(
     *,
     field: StrField | None = None,
     limit: int | None = 20,
-    return_queryset: bool = False,
     case_sensitive: bool = False,
-    synonyms_field: StrField | None = "synonyms",
-) -> pd.DataFrame | QuerySet:
+) -> QuerySet:
     """{}."""
     return _search(
         cls=cls,
         string=string,
         field=field,
-        return_queryset=return_queryset,
         limit=limit,
         case_sensitive=case_sensitive,
-        synonyms_field=synonyms_field,
     )
 
 
@@ -535,7 +487,13 @@ def save(self, *args, **kwargs) -> Registry:
             self_on_db._state.db = db
             self_on_db.pk = pk_on_db
             # by default, transfer parents of the labels to maintain ontological hierarchy
-            add_from_kwargs = {"parents": kwargs.get("parents", True)}
+            try:
+                import bionty as bt
+
+                parents = kwargs.get("parents", bt.settings.auto_save_parents)
+            except ImportError:
+                parents = kwargs.get("parents", True)
+            add_from_kwargs = {"parents": parents}
             logger.info("transfer features")
             self.features._add_from(self_on_db, **add_from_kwargs)
             logger.info("transfer labels")

lamindb/_save.py

@@ -9,10 +9,10 @@ from functools import partial
 from typing import TYPE_CHECKING, Iterable, overload
 
 import lamindb_setup
-from django.db import transaction
+from django.db import IntegrityError, transaction
 from django.utils.functional import partition
 from lamin_utils import logger
-from lamindb_setup.core.upath import print_hook
+from lamindb_setup.core.upath import LocalPathClasses
 from lnschema_core.models import Artifact, Registry
 
 from lamindb.core._settings import settings
@@ -78,14 +78,15 @@ def save(
     # for artifacts, we want to bulk-upload rather than upload one-by-one
     non_artifacts, artifacts = partition(lambda r: isinstance(r, Artifact), records)
     if non_artifacts:
-        # first save all records that do not yet have a primary key without
-        # recursing parents
-        _, non_artifacts_without_pk = partition(lambda r: r.pk is None, non_artifacts)
-        bulk_create(non_artifacts_without_pk, ignore_conflicts=ignore_conflicts)
+        non_artifacts_old, non_artifacts_new = partition(
+            lambda r: r._state.adding or r.pk is None, non_artifacts
+        )
+        bulk_create(non_artifacts_new, ignore_conflicts=ignore_conflicts)
+        if non_artifacts_old:
+            bulk_update(non_artifacts_old)
         non_artifacts_with_parents = [
-            r for r in non_artifacts_without_pk if hasattr(r, "_parents")
+            r for r in non_artifacts_new if hasattr(r, "_parents")
         ]
-
         if len(non_artifacts_with_parents) > 0 and kwargs.get("parents") is not False:
             # this can only happen within lnschema_bionty right now!!
             # we might extend to core lamindb later
@@ -129,6 +130,19 @@ def bulk_create(records: Iterable[Registry], ignore_conflicts: bool | None = Fal
         orm.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
 
 
+def bulk_update(records: Iterable[Registry], ignore_conflicts: bool | None = False):
+    records_by_orm = defaultdict(list)
+    for record in records:
+        records_by_orm[record.__class__].append(record)
+    for orm, records in records_by_orm.items():
+        field_names = [
+            field.name
+            for field in orm._meta.fields
+            if (field.name != "created_at" and field.name != "id")
+        ]
+        orm.objects.bulk_update(records, field_names)
+
+
 # This is also used within Artifact.save()
 def check_and_attempt_upload(
     artifact: Artifact,
@@ -166,9 +180,12 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
     is_dir = local_path.is_dir()
     cache_dir = settings._storage_settings.cache_dir
 
-    # just delete from the cache dir if a local instance
-    if not lamindb_setup.settings.storage.type_is_cloud:
-        if cache_dir in local_path.parents:
+    # just delete from the cache dir if storage_path is local
+    if isinstance(storage_path, LocalPathClasses):
+        if (
+            local_path.as_posix() != storage_path.as_posix()
+            and cache_dir in local_path.parents
+        ):
             if is_dir:
                 shutil.rmtree(local_path)
             else:

lamindb/core/_data.py

@@ -114,6 +114,16 @@ def format_repr(value: Registry, exclude: list[str] | str | None = None) -> str:
 @doc_args(Data.describe.__doc__)
 def describe(self: Data):
     """{}."""
+    # prefetch all many-to-many relationships
+    # doesn't work for describing using artifact
+    # self = (
+    #     self.__class__.objects.using(self._state.db)
+    #     .prefetch_related(
+    #         *[f.name for f in self.__class__._meta.get_fields() if f.many_to_many]
+    #     )
+    #     .get(id=self.id)
+    # )
+
     model_name = self.__class__.__name__
     msg = ""
 
@@ -125,6 +135,19 @@ def describe(self: Data):
             foreign_key_fields.append(f.name)
         else:
             direct_fields.append(f.name)
+    if not self._state.adding:
+        # prefetch foreign key relationships
+        self = (
+            self.__class__.objects.using(self._state.db)
+            .select_related(*foreign_key_fields)
+            .get(id=self.id)
+        )
+        # prefetch m-2-m relationships
+        self = (
+            self.__class__.objects.using(self._state.db)
+            .prefetch_related("feature_sets", "input_of")
+            .get(id=self.id)
+        )
 
     # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
@@ -194,7 +217,7 @@ def get_labels(
             )
         else:
             qs_by_registry[registry] = getattr(
-                self, self.features._accessor_by_orm[registry]
+                self, self.features.accessor_by_orm[registry]
             ).all()
     if flat_names:
         # returns a flat list of names
@@ -282,7 +305,7 @@ def add_labels(
             )
         for registry_name, records in records_by_registry.items():
             labels_accessor = getattr(
-                self, self.features._accessor_by_orm[registry_name]
+                self, self.features.accessor_by_orm[registry_name]
             )
             # remove labels that are already linked as add doesn't perform update
             linked_labels = [r for r in records if r in labels_accessor.filter()]
@@ -321,7 +344,7 @@ def add_labels(
                     found_feature = True
             if not found_feature:
                 if "external" in linked_features_by_slot:
-                    feature_set = self.features._feature_set_by_slot["external"]
+                    feature_set = self.features.feature_set_by_slot["external"]
                     features_list = feature_set.features.list()
                 else:
                     features_list = []

lamindb/core/_feature_manager.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from itertools import compress
-from typing import TYPE_CHECKING, Iterable, Optional
+from typing import TYPE_CHECKING, Iterable
 
 import anndata as ad
 from anndata import AnnData
@@ -57,15 +57,13 @@ def get_feature_set_by_slot(host) -> dict:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     # otherwise, we need a query
-    feature_set_links = host.feature_sets.through.objects.using(host_db).filter(
-        **kwargs
+    feature_set_links = (
+        host.feature_sets.through.objects.using(host_db)
+        .filter(**kwargs)
+        .select_related("feature_set")
     )
-    return {
-        feature_set_link.slot: FeatureSet.objects.using(host_db).get(
-            id=feature_set_link.feature_set_id
-        )
-        for feature_set_link in feature_set_links
-    }
+
+    return {fsl.slot: fsl.feature_set for fsl in feature_set_links}
 
 
 def get_label_links(
@@ -74,7 +72,7 @@ def get_label_links(
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id, "feature_id": feature.id}
     link_records = (
-        getattr(host, host.features._accessor_by_orm[registry])
+        getattr(host, host.features.accessor_by_orm[registry])
         .through.objects.using(host._state.db)
         .filter(**kwargs)
     )
@@ -93,48 +91,50 @@ def print_features(self: Data) -> str:
 
     from ._data import format_repr
 
-    msg = ""
-    features_lookup = Feature.objects.using(self._state.db).lookup().dict()
-    for slot, feature_set in self.features._feature_set_by_slot.items():
+    messages = []
+    for slot, feature_set in get_feature_set_by_slot(self).items():
         if feature_set.registry != "core.Feature":
             features = feature_set.members
+            # features.first() is a lot slower than features[0] here
             name_field = get_default_str_field(features[0])
-            feature_names = [getattr(feature, name_field) for feature in features]
-            msg += (
+            feature_names = list(features.values_list(name_field, flat=True)[:30])
+            messages.append(
                 f"  {colors.bold(slot)}: {format_repr(feature_set, exclude='hash')}\n"
             )
             print_values = _print_values(feature_names, n=20)
-            msg += f"    {print_values}\n"
+            messages.append(f"    {print_values}\n")
         else:
-            df_slot = feature_set.features.df()
-            msg += (
+            features_lookup = {
+                f.name: f for f in Feature.objects.using(self._state.db).filter().all()
+            }
+            messages.append(
                 f"  {colors.bold(slot)}: {format_repr(feature_set, exclude='hash')}\n"
             )
-            for _, row in df_slot.iterrows():
-                if row["type"] == "category" and row["registries"] is not None:
-                    labels = self.labels.get(
-                        features_lookup.get(row["name"]), mute=True
-                    )
+            for name, row_type, registries in feature_set.features.values_list(
+                "name", "type", "registries"
+            ):
+                if row_type == "category" and registries is not None:
+                    labels = self.labels.get(features_lookup.get(name), mute=True)
                     indent = ""
                     if isinstance(labels, dict):
-                        msg += f"    🔗 {row['name']} ({row.registries})\n"
+                        messages.append(f"    🔗 {name} ({registries})\n")
                         indent = "    "
                     else:
-                        labels = {row["registries"]: labels}
-                    for registry, labels in labels.items():  # noqa: B020
-                        count_str = f"{len(labels)}, {colors.italic(f'{registry}')}"
-                        field = get_default_str_field(labels)
-                        print_values = _print_values(labels.list(field), n=10)
+                        labels = {registries: labels}
+                    for registry, registry_labels in labels.items():
+                        field = get_default_str_field(registry_labels)
+                        values_list = registry_labels.values_list(field, flat=True)
+                        count_str = f"{feature_set.n}, {colors.italic(f'{registry}')}"
+                        print_values = _print_values(values_list[:20], n=10)
                         msg_objects = (
-                            f"{indent}    🔗 {row['name']} ({count_str}):"
-                            f" {print_values}\n"
+                            f"{indent}    🔗 {name} ({count_str}):" f" {print_values}\n"
                         )
-                        msg += msg_objects
+                        messages.append(msg_objects)
                 else:
-                    msg += f"    {row['name']} ({row['type']})\n"
-    if msg != "":
-        msg = f"{colors.green('Features')}:\n" + msg
-    return msg
+                    messages.append(f"    {name} ({row_type})\n")
+    if messages:
+        messages.insert(0, f"{colors.green('Features')}:\n")
+    return "".join(messages)
 
 
 def parse_feature_sets_from_anndata(
@@ -204,30 +204,44 @@ class FeatureManager:
 
     def __init__(self, host: Artifact | Collection):
         self._host = host
-        self._feature_set_by_slot = get_feature_set_by_slot(host)
-        self._accessor_by_orm = get_accessor_by_orm(host)
+        self._feature_set_by_slot = None
+        self._accessor_by_orm = None
 
     def __repr__(self) -> str:
-        if len(self._feature_set_by_slot) > 0:
+        if len(self.feature_set_by_slot) > 0:
             return print_features(self._host)
         else:
             return "no linked features"
 
     def __getitem__(self, slot) -> QuerySet:
-        if slot not in self._feature_set_by_slot:
+        if slot not in self.feature_set_by_slot:
             raise ValueError(
                 f"No linked feature set for slot: {slot}\nDid you get validation"
                 " warnings? Only features that match registered features get validated"
                 " and linked."
             )
-        feature_set = self._feature_set_by_slot[slot]
+        feature_set = self.feature_set_by_slot[slot]
         orm_name = feature_set.registry
         if hasattr(feature_set, "_features"):
             # feature set is not yet saved
             # need to think about turning this into a queryset
             return feature_set._features
         else:
-            return getattr(feature_set, self._accessor_by_orm[orm_name]).all()
+            return getattr(feature_set, self.accessor_by_orm[orm_name]).all()
+
+    @property
+    def feature_set_by_slot(self):
+        """Feature sets by slot."""
+        if self._feature_set_by_slot is None:
+            self._feature_set_by_slot = get_feature_set_by_slot(self._host)
+        return self._feature_set_by_slot
+
+    @property
+    def accessor_by_orm(self):
+        """Accessor by ORM."""
+        if self._accessor_by_orm is None:
+            self._accessor_by_orm = get_accessor_by_orm(self._host)
+        return self._accessor_by_orm
 
     def add(self, features: Iterable[Registry], slot: str | None = None):
         """Add features stratified by slot."""
@@ -351,26 +365,36 @@ class FeatureManager:
         )
         if link_record is None:
             self._host.feature_sets.through(**kwargs).save(using=host_db)
-            self._feature_set_by_slot[slot] = feature_set
+            if slot in self.feature_set_by_slot:
+                logger.warning(f"replaced existing {slot} featureset")
+            # this _feature_set_by_slot here is private
+            self._feature_set_by_slot[slot] = feature_set  # type: ignore
 
     def _add_from(self, data: Data, parents: bool = True):
         """Transfer features from a artifact or collection."""
         using_key = settings._using_key
-        for slot, feature_set in data.features._feature_set_by_slot.items():
+        for slot, feature_set in data.features.feature_set_by_slot.items():
+            print(slot)
             members = feature_set.members
-            if members.count() == 0:
+            if len(members) == 0:
                 continue
             registry = members[0].__class__
             # note here the features are transferred based on an unique field
             field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
+            # TODO: get a default ID field for the registry
             if hasattr(registry, "ontology_id") and parents:
                 field = "ontology_id"
+            elif hasattr(registry, "ensembl_gene_id"):
+                field = "ensembl_gene_id"
+            elif hasattr(registry, "uniprotkb_id"):
+                field = "uniprotkb_id"
+
             if registry.__get_name_with_schema__() == "bionty.Organism":
                 parents = False
             # this will be e.g. be a list of ontology_ids or uids
             member_uids = list(members.values_list(field, flat=True))
             # create records from ontology_id in order to populate parents
-            if field == "ontology_id" and len(member_uids) > 0:
+            if field == "ontology_id" and len(member_uids) > 0 and parents:
                 # create from bionty
                 records = registry.from_values(member_uids, field=field)
                 if len(records) > 0:
@@ -378,8 +402,9 @@ class FeatureManager:
             validated = registry.validate(member_uids, field=field, mute=True)
             new_members_uids = list(compress(member_uids, ~validated))
             new_members = members.filter(**{f"{field}__in": new_members_uids}).all()
-            if new_members.count() > 0:
-                mute = True if new_members.count() > 10 else False
+            n_new_members = len(new_members)
+            if n_new_members > 0:
+                mute = True if n_new_members > 10 else False
                 # transfer foreign keys needs to be run before transfer to default db
                 transfer_fk_to_default_db_bulk(new_members, using_key)
                 for feature in new_members:
@@ -390,9 +415,7 @@ class FeatureManager:
                     transfer_to_default_db(
                         feature, using_key, mute=mute, transfer_fk=False
                     )
-                logger.info(
-                    f"saving {new_members.count()} new {registry.__name__} records"
-                )
+                logger.info(f"saving {n_new_members} new {registry.__name__} records")
                 save(new_members, parents=parents)
 
             # create a new feature set from feature values using the same uid

lamindb/core/_label_manager.py

@@ -42,14 +42,17 @@ def get_labels_as_dict(self: Data):
     return labels
 
 
-def print_labels(self: Data):
+def print_labels(self: Data, field: str = "name"):
     labels_msg = ""
     for related_name, (related_model, labels) in get_labels_as_dict(self).items():
-        if labels.exists():
-            n = labels.count()
-            field = get_default_str_field(labels)
-            print_values = _print_values(labels.list(field), n=10)
-            labels_msg += f"  📎 {related_name} ({n}, {colors.italic(related_model)}): {print_values}\n"
+        try:
+            labels_list = list(labels.values_list(field, flat=True))
+            if len(labels_list) > 0:
+                get_default_str_field(labels)
+                print_values = _print_values(labels_list[:20], n=10)
+                labels_msg += f"  📎 {related_name} ({len(labels_list)}, {colors.italic(related_model)}): {print_values}\n"
+        except Exception:
+            continue
     if len(labels_msg) > 0:
         return f"{colors.green('Labels')}:\n{labels_msg}"
     else:
@@ -72,7 +75,7 @@ def transfer_add_labels(labels, features_lookup_self, self, row, parents: bool =
         # link labels records from self db
         self._host.labels.add(
             validated_labels + new_labels,
-            feature=getattr(features_lookup_self, row["name"]),
+            feature=features_lookup_self.get(row["name"]),
         )
 
     # validate labels on the default db
@@ -94,6 +97,10 @@ def validate_labels(labels: QuerySet | list | dict, parents: bool = True):
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
         if hasattr(registry, "ontology_id") and parents:
             field = "ontology_id"
+        elif hasattr(registry, "ensembl_gene_id"):
+            field = "ensembl_gene_id"
+        elif hasattr(registry, "uniprotkb_id"):
+            field = "uniprotkb_id"
         if registry.__get_name_with_schema__() == "bionty.Organism":
             parents = False
         # if the field value is None, use uid field
@@ -195,9 +202,13 @@ class LabelManager:
             >>> file1.ulabels.set(labels)
             >>> file2.labels.add_from(file1)
         """
-        features_lookup_self = Feature.lookup()
-        features_lookup_data = Feature.objects.using(data._state.db).lookup()
-        for _, feature_set in data.features._feature_set_by_slot.items():
+        from django.db.utils import ProgrammingError
+
+        features_lookup_self = {f.name: f for f in Feature.objects.filter().all()}
+        features_lookup_data = {
+            f.name: f for f in Feature.objects.using(data._state.db).filter().all()
+        }
+        for _, feature_set in data.features.feature_set_by_slot.items():
             # add labels stratified by feature
             if feature_set.registry == "core.Feature":
                 # df_slot is the Feature table with type and registries
@@ -207,30 +218,31 @@ class LabelManager:
                         logger.info(f"transferring {row['name']}")
                         # labels records from data db
                         labels = data.labels.get(
-                            getattr(features_lookup_data, row["name"]), mute=True
+                            features_lookup_data.get(row["name"]), mute=True
                         )
                         transfer_add_labels(
                             labels, features_lookup_self, self, row, parents=parents
                         )
-
-        # for now, have this be duplicated, need to disentangle above
+        # TODO: for now, has to be duplicated
         using_key = settings._using_key
         for related_name, (_, labels) in get_labels_as_dict(data).items():
             labels = labels.all()
-            if len(labels) == 0:
+            try:
+                if len(labels) == 0:
+                    continue
+                validated_labels, new_labels = validate_labels(labels, parents=parents)
+                if len(new_labels) > 0:
+                    transfer_fk_to_default_db_bulk(new_labels, using_key)
+                    for label in new_labels:
+                        transfer_to_default_db(
+                            label, using_key, mute=True, transfer_fk=False
+                        )
+                    save(new_labels, parents=parents)
+                # this should not occur as file and collection should have the same attributes
+                # but this might not be true for custom schema
+                labels_list = validated_labels + new_labels
+                if hasattr(self._host, related_name):
+                    getattr(self._host, related_name).add(*labels_list)
+            # ProgrammingError is raised when schemas don't match between source and target instances
+            except ProgrammingError:
                 continue
-            validated_labels, new_labels = validate_labels(
-                labels.all(), parents=parents
-            )
-            if len(new_labels) > 0:
-                transfer_fk_to_default_db_bulk(new_labels, using_key)
-                for label in new_labels:
-                    transfer_to_default_db(
-                        label, using_key, mute=True, transfer_fk=False
-                    )
-                save(new_labels, parents=parents)
-            # this should not occur as file and collection should have the same attributes
-            # but this might not be true for custom schema
-            labels_list = validated_labels + new_labels
-            if hasattr(self._host, related_name):
-                getattr(self._host, related_name).add(*labels_list)

{lamindb-0.71.2.dist-info → lamindb-0.71.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.71.2
+Version: 0.71.3
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.66.4
-Requires-Dist: lamindb_setup==0.71.3
+Requires-Dist: lnschema_core==0.66.5
+Requires-Dist: lamindb_setup==0.71.4
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.13.1
+Requires-Dist: lamin_cli==0.13.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,8 +23,9 @@ Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
+Requires-Dist: psutil
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.42.9 ; extra == "bionty"
+Requires-Dist: bionty==0.42.11 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -37,7 +38,7 @@ Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
-Requires-Dist: nbproject==0.10.2 ; extra == "jupyter"
+Requires-Dist: nbproject==0.10.3 ; extra == "jupyter"
 Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"

{lamindb-0.71.2.dist-info → lamindb-0.71.3.dist-info}/RECORD

@@ -1,29 +1,29 @@
-lamindb/__init__.py,sha256=GKrW6unkqBBwwpxTXjuUv-5k4c4unimsV-vGFSWt68I,2182
+lamindb/__init__.py,sha256=qomw1l2wiK8BNFYinT6ejwDpCRMTtD9EHY3678R253k,2182
 lamindb/_annotate.py,sha256=kgbilILfgzoS-GEpjxzVwRMs7CoSa9BNEcIWXFBW69I,43915
-lamindb/_artifact.py,sha256=8uBW-dhuWyBUQGs728sAPCnuhTic-NKjSbaneF07aMo,40106
-lamindb/_can_validate.py,sha256=nvoZG-35n3HofkY4Xc6hBv9AV54_RDan7Hzp5TuqY9I,14709
-lamindb/_collection.py,sha256=SDM35R_5WHrgLKjVb14Q8-Rz_gn5hdZLJobPcanm4PM,14627
+lamindb/_artifact.py,sha256=Xm3Q0yLQxKHkYHUVR7Tlg6SIRVyg6hf7DSXhoilsEQM,40509
+lamindb/_can_validate.py,sha256=s1q0lxplqnhytrVgArBTm05XKMMmpreK0ZlVCsd2jjk,14849
+lamindb/_collection.py,sha256=T6_jVVdYp4ewAlTu40NoH7NhgwOWJYM4jNsK51JE_Wg,14624
 lamindb/_feature.py,sha256=srAKchY7gqD-h-cWlEiAWuHlpFKFwv0PWIA-JX0Go8c,6758
 lamindb/_feature_set.py,sha256=AzjOcHzQajpeikPOAic-aj0z_C5b7VpHVegg3ThRSLw,9045
 lamindb/_filter.py,sha256=xnjJzjF3Zj4dK_Kfymvhgczk27MhhXz5ZYc7XINbgHY,1331
 lamindb/_finish.py,sha256=6GwhqrC-x-JdFd16i7-uyhCWeQgGKxr25aSsSXPZt4g,8598
-lamindb/_from_values.py,sha256=DVXjnQ2wwNw-2bFzy0uXLdVlqoprrn95hTnrXwn-KqM,12638
+lamindb/_from_values.py,sha256=L9RBI9G7TZCOnbT_DcLodhFccG4E9zbmWrhM1P3eKrc,13544
 lamindb/_is_versioned.py,sha256=0PgRCmxEmYDcAjllLSOYZm132B1lW6QgmBBERhRyFt0,1341
 lamindb/_parents.py,sha256=N9T8jbd3eaoHDLE9TD1y1QgGcO81E6Brapy8LILzRCQ,14790
 lamindb/_query_manager.py,sha256=3zokXqxgj9vTJBnN2sbYKS-q69fyDDPF_aGq_rFHzXU,4066
 lamindb/_query_set.py,sha256=n0owd74cTzGz6-mIv8SlDz0wcyRz7Xw3Ke1LhE8UlIg,10784
-lamindb/_registry.py,sha256=fmX-BUnan3Y0WrEAx3qNwRYCIJwJgjoKnRnpgcXujEI,19358
+lamindb/_registry.py,sha256=xgHyw49yRcqxaUdzCZddFgqURBSHHX_kjIHGteLFnP4,18173
 lamindb/_run.py,sha256=We50MUeGH778begutDGoNFM-n5_81_BfMCnZS1bdkt0,1937
-lamindb/_save.py,sha256=_7r3TUV3B6Hp75r5O_ymu3fKWyBHbGa5vmE_pxrtsVI,10923
+lamindb/_save.py,sha256=pksthZrL3SMjLoFGmRNnCO92iXHHHhRk9mpOlW7lXTU,11514
 lamindb/_storage.py,sha256=VW8xq3VRv58-ciholvOdlcgvp_OIlLxx5GxLt-e2Irs,614
 lamindb/_transform.py,sha256=E9C7psuOnsNrUQpWRuGgEUM8_pc7YhDn7n4ieHzB4X0,3169
 lamindb/_ulabel.py,sha256=e5dw9h1tR0_u-DMn7Gzx0WhUhV5w7j4v3QbnLWQV7eI,1941
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=GV1FrqIMmdooEkA-5zvcTWgV1nqx1sehi6WdWEaFpxM,2171
 lamindb/core/__init__.py,sha256=TI9_1Jtpwae_cUPQ3-U0RRPH5c3GBA-gLhHvlAk_Nlo,1213
-lamindb/core/_data.py,sha256=Lico6-Vx15bNpGLl1bqFqEsh62pD4YKOOBnmahse1tI,17673
-lamindb/core/_feature_manager.py,sha256=uTzZZ7-qqEAmdwi48Holy2j5VGTgmoQxhb21r6mLShI,15824
-lamindb/core/_label_manager.py,sha256=0RtegYnK3zIisOnd970EobOrHMpp7OCH-mEoPrPXw2c,9075
+lamindb/core/_data.py,sha256=ujwl2fA0gScz610DN50bAdAi_XCF-USw_yhwXkhWcFY,18445
+lamindb/core/_feature_manager.py,sha256=WvHY1zhB7vg8Dg7JW9ouqi_qAKJOfJ4gklwnHSiEX7s,16833
+lamindb/core/_label_manager.py,sha256=HXWYYg6k6vfsTGgXcfjMbxQTsIRV1a5m-WWsC4s-daU,9699
 lamindb/core/_mapped_collection.py,sha256=_OwFZh5SePDUD70XIK5kngv3we_Z5-YdGHNfpUSatSQ,19469
 lamindb/core/_run_context.py,sha256=7iCCOB2z154puBI7ZKzcaEZ5l6_9S8aSYBOBJI65lyc,17117
 lamindb/core/_settings.py,sha256=rW1KfEXfT56XErwcnSuQxaCytpOy1kJ-u7tVmkmNmxY,6131
@@ -48,7 +48,7 @@ lamindb/integrations/__init__.py,sha256=aH2PmO2m4-vwIifMYTB0Fyyr_gZWtVnV71jT0tVW
 lamindb/integrations/_vitessce.py,sha256=b0FqTBsP-M6Q7xCYXVwFwM8DOIeeOBZEhYbryhtq4gk,2535
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.71.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.71.2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.71.2.dist-info/METADATA,sha256=l49_xPwqfUDB6jUvUQoAVeQu8Tj3JUNCfTPB9cqOq_Y,2674
-lamindb-0.71.2.dist-info/RECORD,,
+lamindb-0.71.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.71.3.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.71.3.dist-info/METADATA,sha256=ITpf4eh_xlvgR5d0Z_hn8FSJ8s7JUsZRX4fMNoQ1bDA,2697
+lamindb-0.71.3.dist-info/RECORD,,