lamindb 0.71.2__py3-none-any.whl → 0.72.0__py3-none-any.whl

lamindb/_from_values.py

@@ -185,8 +185,15 @@ def create_records_from_public(
 
     # create the corresponding bionty object from model
     try:
+        # TODO: more generic
+        organism = kwargs.get("organism")
+        if field.field.name == "ensembl_gene_id":
+            if iterable_idx[0].startswith("ENSG"):
+                organism = "human"
+            elif iterable_idx[0].startswith("ENSMUSG"):
+                organism = "mouse"
         public_ontology = model.public(
-            organism=kwargs.get("organism"), public_source=kwargs.get("public_source")
+            organism=organism, public_source=kwargs.get("public_source")
         )
     except Exception:
         # for custom records that are not created from public sources
@@ -223,8 +230,15 @@ def create_records_from_public(
         bionty_kwargs, multi_msg = _bulk_create_dicts_from_df(
             keys=mapped_values, column_name=field.field.name, df=bionty_df
         )
+        organism_kwargs = {}
+        if "organism" not in kwargs:
+            organism_record = _get_organism_record(
+                field, public_ontology.organism, force=True
+            )
+            if organism_record is not None:
+                organism_kwargs["organism"] = organism_record
         for bk in bionty_kwargs:
-            records.append(model(**bk, **kwargs))
+            records.append(model(**bk, **kwargs, **organism_kwargs))
 
         # number of records that matches field (not synonyms)
         validated = result.validated
@@ -260,10 +274,11 @@ def index_iterable(iterable: Iterable) -> pd.Index:
     return idx[(idx != "") & (~idx.isnull())]
 
 
-def _print_values(names: list, n: int = 20) -> str:
-    names = list(set(names))
-    print_values = ", ".join([f"'{name}'" for name in names[:n] if name != "None"])
-    if len(names) > n:
+def _print_values(names: Iterable, n: int = 20) -> str:
+    names = (name for name in names if name != "None")
+    unique_names = list(dict.fromkeys(names))[:n]
+    print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if len(unique_names) > n:
         print_values += ", ..."
     return print_values
 
@@ -334,9 +349,13 @@ def _has_organism_field(orm: Registry) -> bool:
         return False
 
 
-def _get_organism_record(field: StrField, organism: str | Registry) -> Registry:
+def _get_organism_record(
+    field: StrField, organism: str | Registry, force: bool = False
+) -> Registry:
     model = field.field.model
-    if _has_organism_field(model):
+    check = True if force else field.field.name != "ensembl_gene_id"
+
+    if _has_organism_field(model) and check:
         from lnschema_bionty._bionty import create_or_get_organism_record
 
         organism_record = create_or_get_organism_record(organism=organism, orm=model)

lamindb/_query_manager.py

@@ -7,6 +7,8 @@ from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Registry
 
+from lamindb.core._settings import settings
+
 from .core._feature_manager import get_feature_set_by_slot
 
 if TYPE_CHECKING:
@@ -41,7 +43,10 @@ class QueryManager(models.Manager):
                 from lamindb.core._data import WARNING_RUN_TRANSFORM, _track_run_input
                 from lamindb.core._run_context import run_context
 
-                if run_context.run is None:
+                if (
+                    run_context.run is None
+                    and not settings.silence_file_run_transform_warning
+                ):
                     logger.warning(WARNING_RUN_TRANSFORM)
                 _track_run_input(self.instance)
 

lamindb/_registry.py

@@ -2,14 +2,12 @@ from __future__ import annotations
 
 import builtins
 from typing import TYPE_CHECKING, Iterable, List, NamedTuple
-from uuid import UUID
 
 import dj_database_url
 import lamindb_setup as ln_setup
-import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from django.db import connections
-from django.db.models import Manager, QuerySet
+from django.db.models import Manager, Q, QuerySet
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
 from lamin_utils._search import search as base_search
@@ -22,19 +20,17 @@ from lnschema_core import Registry
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._settings import settings
+from lamindb.core.exceptions import ValidationError
 
 from ._from_values import get_or_create_records
 
 if TYPE_CHECKING:
+    import pandas as pd
     from lnschema_core.types import ListLike, StrField
 
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
 
 
-class ValidationError(Exception):
-    pass
-
-
 def init_self_from_db(self: Registry, existing_record: Registry):
     new_args = [
         getattr(existing_record, field.attname) for field in self._meta.concrete_fields
@@ -61,20 +57,15 @@ def suggest_objects_with_same_name(orm: Registry, kwargs) -> str | None:
     if kwargs.get("name") is None:
         return None
     else:
-        results = orm.search(kwargs["name"])
-        if results.shape[0] == 0:
+        queryset = orm.search(kwargs["name"])
+        if not queryset.exists():  # empty queryset
             return None
-
-        # subset results to those with at least 0.90 levensteihn distance
-        results = results.loc[results.score >= 90]
-
-        # test for exact match
-        if len(results) > 0:
-            if results.index[0] == kwargs["name"]:
-                return "object-with-same-name-exists"
+        else:
+            for record in queryset:
+                if record.name == kwargs["name"]:
+                    return "object-with-same-name-exists"
             else:
-                s = "" if results.shape[0] == 1 else "s"
-                it = "it" if results.shape[0] == 1 else "one of them"
+                s, it = ("", "it") if len(queryset) == 1 else ("s", "one of them")
                 msg = (
                     f"record{s} with similar name{s} exist! did you mean to load {it}?"
                 )
@@ -83,9 +74,9 @@ def suggest_objects_with_same_name(orm: Registry, kwargs) -> str | None:
 
                     logger.warning(f"{msg}")
                     if settings._verbosity_int >= 1:
-                        display(results)
+                        display(queryset.df())
                 else:
-                    logger.warning(f"{msg}\n{results}")
+                    logger.warning(f"{msg}\n{queryset}")
     return None
 
 
@@ -162,80 +153,42 @@ def _search(
     string: str,
     *,
     field: StrField | list[StrField] | None = None,
-    limit: int | None = 10,
-    return_queryset: bool = False,
+    limit: int | None = 20,
     case_sensitive: bool = False,
-    synonyms_field: StrField | None = "synonyms",
     using_key: str | None = None,
-) -> pd.DataFrame | QuerySet:
-    queryset = _queryset(cls, using_key=using_key)
-    orm = queryset.model
-
-    def _search_single_field(
-        string: str,
-        field: StrField | None,
-        synonyms_field: StrField | None = "synonyms",
-    ) -> pd.DataFrame:
-        field = get_default_str_field(orm=orm, field=field)
-
-        try:
-            orm._meta.get_field(synonyms_field)
-            synonyms_field_exists = True
-        except FieldDoesNotExist:
-            synonyms_field_exists = False
-
-        if synonyms_field is not None and synonyms_field_exists:
-            df = pd.DataFrame(queryset.values("uid", field, synonyms_field))
-        else:
-            df = pd.DataFrame(queryset.values("uid", field))
-
-        return base_search(
-            df=df,
-            string=string,
-            field=field,
-            limit=limit,
-            synonyms_field=str(synonyms_field),
-            case_sensitive=case_sensitive,
-        )
-
-    # search in both key and description fields for Artifact
-    if orm._meta.model.__name__ == "Artifact" and field is None:
-        field = ["key", "description"]
-
-    if not isinstance(field, List):
-        field = [field]
-
-    results = []
-    for fd in field:
-        result_field = _search_single_field(
-            string=string, field=fd, synonyms_field=synonyms_field
-        )
-        results.append(result_field)
-        # turn off synonyms search after the 1st field
-        synonyms_field = None
-
-    if len(results) > 1:
-        result = (
-            pd.concat([r.reset_index() for r in results], join="outer")
-            .drop(columns=["index"], errors="ignore")
-            .set_index("uid")
-        )
-    else:
-        result = results[0]
-
-    # remove results that have __ratio__ 0
-    if "__ratio__" in result.columns:
-        result = result[result["__ratio__"] > 0].sort_values(
-            "__ratio__", ascending=False
-        )
-        # restrict to 1 decimal
-        # move the score to be the last column
-        result["score"] = result.pop("__ratio__").round(1)
-
-    if return_queryset:
-        return _order_queryset_by_ids(queryset, result.reset_index()["uid"])
+) -> QuerySet:
+    input_queryset = _queryset(cls, using_key=using_key)
+    orm = input_queryset.model
+    if field is None:
+        fields = [
+            field.name
+            for field in orm._meta.fields
+            if field.get_internal_type() in {"CharField", "TextField"}
+        ]
     else:
-        return result.fillna("")
+        if not isinstance(field, list):
+            fields_input = [field]
+        else:
+            fields_input = field
+        fields = []
+        for field in fields_input:
+            if not isinstance(field, str):
+                try:
+                    fields.append(field.field.name)
+                except AttributeError as error:
+                    raise TypeError(
+                        "Please pass a Registry string field, e.g., `CellType.name`!"
+                    ) from error
+            else:
+                fields.append(field)
+    expression = Q()
+    case_sensitive_i = "" if case_sensitive else "i"
+    for field in fields:
+        # Construct the keyword for the Q object dynamically
+        query = {f"{field}__{case_sensitive_i}contains": string}
+        expression |= Q(**query)  # Unpack the dictionary into Q()
+    output_queryset = input_queryset.filter(expression)[:limit]
+    return output_queryset
 
 
 @classmethod  # type: ignore
@@ -246,19 +199,15 @@ def search(
     *,
     field: StrField | None = None,
     limit: int | None = 20,
-    return_queryset: bool = False,
     case_sensitive: bool = False,
-    synonyms_field: StrField | None = "synonyms",
-) -> pd.DataFrame | QuerySet:
+) -> QuerySet:
     """{}."""
     return _search(
         cls=cls,
         string=string,
         field=field,
-        return_queryset=return_queryset,
         limit=limit,
         case_sensitive=case_sensitive,
-        synonyms_field=synonyms_field,
     )
 
 
@@ -470,7 +419,8 @@ def transfer_to_default_db(
             if run_context.run is not None:
                 record.run_id = run_context.run.id
             else:
-                logger.warning(WARNING_RUN_TRANSFORM)
+                if not settings.silence_file_run_transform_warning:
+                    logger.warning(WARNING_RUN_TRANSFORM)
                 record.run_id = None
         if hasattr(record, "transform_id") and record._meta.model_name != "run":
             record.transform = None
@@ -535,7 +485,13 @@ def save(self, *args, **kwargs) -> Registry:
             self_on_db._state.db = db
             self_on_db.pk = pk_on_db
             # by default, transfer parents of the labels to maintain ontological hierarchy
-            add_from_kwargs = {"parents": kwargs.get("parents", True)}
+            try:
+                import bionty as bt
+
+                parents = kwargs.get("parents", bt.settings.auto_save_parents)
+            except ImportError:
+                parents = kwargs.get("parents", True)
+            add_from_kwargs = {"parents": parents}
             logger.info("transfer features")
             self.features._add_from(self_on_db, **add_from_kwargs)
             logger.info("transfer labels")
@@ -575,7 +531,11 @@ def __get_schema_name__(cls) -> str:
 @classmethod  # type: ignore
 def __get_name_with_schema__(cls) -> str:
     schema_name = cls.__get_schema_name__()
-    return f"{schema_name}.{cls.__name__}"
+    if schema_name == "core":
+        schema_prefix = ""
+    else:
+        schema_prefix = f"{schema_name}."
+    return f"{schema_prefix}{cls.__name__}"
 
 
 Registry.__get_schema_name__ = __get_schema_name__

lamindb/_run.py

@@ -13,7 +13,6 @@ def __init__(run: Run, *args, **kwargs):
     transform: Transform = None
     if "transform" in kwargs or len(args) == 1:
         transform = kwargs.pop("transform") if len(args) == 0 else args[0]
-    params: str | None = kwargs.pop("params") if "params" in kwargs else None
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
     reference_type: str | None = (
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
@@ -26,7 +25,6 @@ def __init__(run: Run, *args, **kwargs):
         transform=transform,
         reference=reference,
         reference_type=reference_type,
-        json=params,
     )
 
 

lamindb/_save.py

@@ -9,10 +9,10 @@ from functools import partial
 from typing import TYPE_CHECKING, Iterable, overload
 
 import lamindb_setup
-from django.db import transaction
+from django.db import IntegrityError, transaction
 from django.utils.functional import partition
 from lamin_utils import logger
-from lamindb_setup.core.upath import print_hook
+from lamindb_setup.core.upath import LocalPathClasses
 from lnschema_core.models import Artifact, Registry
 
 from lamindb.core._settings import settings
@@ -78,14 +78,15 @@ def save(
     # for artifacts, we want to bulk-upload rather than upload one-by-one
     non_artifacts, artifacts = partition(lambda r: isinstance(r, Artifact), records)
     if non_artifacts:
-        # first save all records that do not yet have a primary key without
-        # recursing parents
-        _, non_artifacts_without_pk = partition(lambda r: r.pk is None, non_artifacts)
-        bulk_create(non_artifacts_without_pk, ignore_conflicts=ignore_conflicts)
+        non_artifacts_old, non_artifacts_new = partition(
+            lambda r: r._state.adding or r.pk is None, non_artifacts
+        )
+        bulk_create(non_artifacts_new, ignore_conflicts=ignore_conflicts)
+        if non_artifacts_old:
+            bulk_update(non_artifacts_old)
         non_artifacts_with_parents = [
-            r for r in non_artifacts_without_pk if hasattr(r, "_parents")
+            r for r in non_artifacts_new if hasattr(r, "_parents")
         ]
-
         if len(non_artifacts_with_parents) > 0 and kwargs.get("parents") is not False:
             # this can only happen within lnschema_bionty right now!!
             # we might extend to core lamindb later
@@ -129,6 +130,19 @@ def bulk_create(records: Iterable[Registry], ignore_conflicts: bool | None = Fal
         orm.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
 
 
+def bulk_update(records: Iterable[Registry], ignore_conflicts: bool | None = False):
+    records_by_orm = defaultdict(list)
+    for record in records:
+        records_by_orm[record.__class__].append(record)
+    for orm, records in records_by_orm.items():
+        field_names = [
+            field.name
+            for field in orm._meta.fields
+            if (field.name != "created_at" and field.name != "id")
+        ]
+        orm.objects.bulk_update(records, field_names)
+
+
 # This is also used within Artifact.save()
 def check_and_attempt_upload(
     artifact: Artifact,
@@ -166,9 +180,12 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
     is_dir = local_path.is_dir()
     cache_dir = settings._storage_settings.cache_dir
 
-    # just delete from the cache dir if a local instance
-    if not lamindb_setup.settings.storage.type_is_cloud:
-        if cache_dir in local_path.parents:
+    # just delete from the cache dir if storage_path is local
+    if isinstance(storage_path, LocalPathClasses):
+        if (
+            local_path.as_posix() != storage_path.as_posix()
+            and cache_dir in local_path.parents
+        ):
             if is_dir:
                 shutil.rmtree(local_path)
             else:

lamindb/core/__init__.py

@@ -15,6 +15,8 @@ Registries:
    IsVersioned
    CanValidate
    HasParents
+   TracksRun
+   TracksUpdates
    InspectResult
    fields
 
@@ -56,6 +58,8 @@ from lnschema_core.models import (
     HasParents,
     IsVersioned,
     Registry,
+    TracksRun,
+    TracksUpdates,
 )
 
 from lamindb._annotate import (

lamindb/core/_data.py

@@ -18,10 +18,6 @@ from lnschema_core.models import (
     format_field_value,
 )
 
-from lamindb._feature_set import (
-    dict_related_model_to_related_name,
-    dict_schema_name_to_model_name,
-)
 from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
 from lamindb.core._settings import settings
@@ -36,6 +32,10 @@ from ._feature_manager import (
 from ._label_manager import LabelManager, print_labels
 from ._run_context import run_context
 from .exceptions import ValidationError
+from .schema import (
+    dict_related_model_to_related_name,
+    dict_schema_name_to_model_name,
+)
 
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
@@ -87,7 +87,7 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
         for slot, feature_set in self._feature_sets.items():
             kwargs = {
                 host_id_field: self.id,
-                "feature_set_id": feature_set.id,
+                "featureset_id": feature_set.id,
                 "slot": slot,
             }
             links.append(Data.feature_sets.through(**kwargs))
@@ -114,6 +114,16 @@ def format_repr(value: Registry, exclude: list[str] | str | None = None) -> str:
 @doc_args(Data.describe.__doc__)
 def describe(self: Data):
     """{}."""
+    # prefetch all many-to-many relationships
+    # doesn't work for describing using artifact
+    # self = (
+    #     self.__class__.objects.using(self._state.db)
+    #     .prefetch_related(
+    #         *[f.name for f in self.__class__._meta.get_fields() if f.many_to_many]
+    #     )
+    #     .get(id=self.id)
+    # )
+
     model_name = self.__class__.__name__
     msg = ""
 
@@ -125,6 +135,19 @@ def describe(self: Data):
             foreign_key_fields.append(f.name)
         else:
             direct_fields.append(f.name)
+    if not self._state.adding:
+        # prefetch foreign key relationships
+        self = (
+            self.__class__.objects.using(self._state.db)
+            .select_related(*foreign_key_fields)
+            .get(id=self.id)
+        )
+        # prefetch m-2-m relationships
+        self = (
+            self.__class__.objects.using(self._state.db)
+            .prefetch_related("feature_sets", "input_of")
+            .get(id=self.id)
+        )
 
     # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
@@ -152,16 +175,13 @@ def describe(self: Data):
 
 
 def validate_feature(feature: Feature, records: list[Registry]) -> None:
-    """Validate feature record, set feature.registries based on labels records."""
+    """Validate feature record, adjust feature.dtype based on labels records."""
     if not isinstance(feature, Feature):
         raise TypeError("feature has to be of type Feature")
     if feature._state.adding:
         registries = {record.__class__.__get_name_with_schema__() for record in records}
         registries_str = "|".join(registries)
-        msg = (
-            f"ln.Feature(name='{feature.name}', type='category',"
-            f" registries='{registries_str}').save()"
-        )
+        msg = f"ln.Feature(name='{feature.name}', type='cat[{registries_str}]').save()"
         raise ValidationError(f"Feature not validated. If it looks correct: {msg}")
 
 
@@ -174,9 +194,9 @@ def get_labels(
     """{}."""
     if not isinstance(feature, Feature):
         raise TypeError("feature has to be of type Feature")
-    if feature.registries is None:
+    if feature.dtype is None or not feature.dtype.startswith("cat["):
         raise ValueError("feature does not have linked labels")
-    registries_to_check = feature.registries.split("|")
+    registries_to_check = feature.dtype.replace("cat[", "").rstrip("]").split("|")
     if len(registries_to_check) > 1 and not mute:
         logger.warning("labels come from multiple registries!")
     # return an empty query set if self.id is still None
@@ -186,15 +206,15 @@ def get_labels(
     for registry in registries_to_check:
         # currently need to distinguish between ULabel and non-ULabel, because
         # we only have the feature information for Label
-        if registry == "core.ULabel":
+        if registry == "ULabel":
             links_to_labels = get_label_links(self, registry, feature)
             label_ids = [link.ulabel_id for link in links_to_labels]
             qs_by_registry[registry] = ULabel.objects.using(self._state.db).filter(
                 id__in=label_ids
             )
-        else:
+        elif registry in self.features.accessor_by_orm:
             qs_by_registry[registry] = getattr(
-                self, self.features._accessor_by_orm[registry]
+                self, self.features.accessor_by_orm[registry]
             ).all()
     if flat_names:
         # returns a flat list of names
@@ -204,7 +224,7 @@ def get_labels(
         for v in qs_by_registry.values():
             values += v.list(get_default_str_field(v))
         return values
-    if len(registries_to_check) == 1:
+    if len(registries_to_check) == 1 and registry in qs_by_registry:
         return qs_by_registry[registry]
     else:
         return qs_by_registry
@@ -238,9 +258,9 @@ def add_labels(
                 "Please pass a feature, e.g., via: label = ln.ULabel(name='my_label',"
                 " feature=ln.Feature(name='my_feature'))"
             )
-        if feature.registries is not None:
+        if feature.dtype.startswith("cat["):
             orm_dict = dict_schema_name_to_model_name(Artifact)
-            for reg in feature.registries.split("|"):
+            for reg in feature.dtype.replace("cat[", "").rstrip("]").split("|"):
                 orm = orm_dict.get(reg)
                 records_validated += orm.from_values(records, field=field)
 
@@ -281,8 +301,11 @@ def add_labels(
                 record
             )
         for registry_name, records in records_by_registry.items():
+            if registry_name not in self.features.accessor_by_orm:
+                logger.warning(f"skipping {registry_name}")
+                continue
             labels_accessor = getattr(
-                self, self.features._accessor_by_orm[registry_name]
+                self, self.features.accessor_by_orm[registry_name]
             )
             # remove labels that are already linked as add doesn't perform update
             linked_labels = [r for r in records if r in labels_accessor.filter()]
@@ -290,26 +313,29 @@ def add_labels(
                 labels_accessor.remove(*linked_labels)
             labels_accessor.add(*records, through_defaults={"feature_id": feature.id})
         feature_set_links = get_feature_set_links(self)
-        feature_set_ids = [link.feature_set_id for link in feature_set_links.all()]
+        feature_set_ids = [link.featureset_id for link in feature_set_links.all()]
         # get all linked features of type Feature
         feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
         linked_features_by_slot = {
-            feature_set_links.filter(feature_set_id=feature_set.id)
+            feature_set_links.filter(featureset_id=feature_set.id)
             .one()
             .slot: feature_set.features.all()
             for feature_set in feature_sets
-            if "core.Feature" == feature_set.registry
+            if "Feature" == feature_set.registry
         }
         for registry_name, _ in records_by_registry.items():
             msg = ""
-            if feature.registries is None or registry_name not in feature.registries:
+            if (
+                not feature.dtype.startswith("cat[")
+                or registry_name not in feature.dtype
+            ):
                 if len(msg) > 0:
                     msg += ", "
                 msg += f"linked feature '{feature.name}' to registry '{registry_name}'"
-                if feature.registries is None:
-                    feature.registries = registry_name
-                elif registry_name not in feature.registries:
-                    feature.registries += f"|{registry_name}"
+                if not feature.dtype.startswith("cat["):
+                    feature.dtype = f"cat[{registry_name}]"
+                elif registry_name not in feature.dtype:
+                    feature.dtype = feature.dtype.rstrip("]") + f"|{registry_name}]"
                 feature.save()
             if len(msg) > 0:
                 logger.save(msg)
@@ -321,7 +347,7 @@ def add_labels(
                     found_feature = True
             if not found_feature:
                 if "external" in linked_features_by_slot:
-                    feature_set = self.features._feature_set_by_slot["external"]
+                    feature_set = self.features.feature_set_by_slot["external"]
                     features_list = feature_set.features.list()
                 else:
                     features_list = []
@@ -334,11 +360,11 @@ def add_labels(
                     ).one()
                     old_feature_set_link.delete()
                     remaining_links = self.feature_sets.through.objects.filter(
-                        feature_set_id=feature_set.id
+                        featureset_id=feature_set.id
                     ).all()
                     if len(remaining_links) == 0:
                         old_feature_set = FeatureSet.filter(
-                            id=old_feature_set_link.feature_set_id
+                            id=old_feature_set_link.featureset_id
                         ).one()
                         logger.info(
                             "nothing links to it anymore, deleting feature set"