lamindb 0.72.1__py3-none-any.whl → 0.73.1__py3-none-any.whl

lamindb/_query_set.py

@@ -99,9 +99,30 @@ class QuerySet(models.QuerySet, CanValidate):
     @doc_args(Registry.df.__doc__)
     def df(self, include: str | list[str] | None = None) -> pd.DataFrame:
         """{}."""
-        data = self.values()
-        keys = get_keys_from_df(data, self.model)
-        df = pd.DataFrame(self.values(), columns=keys)
+        # re-order the columns
+        exclude_field_names = ["created_at"]
+        field_names = [
+            field.name
+            for field in self.model._meta.fields
+            if (
+                not isinstance(field, models.ForeignKey)
+                and field.name not in exclude_field_names
+            )
+        ]
+        field_names += [
+            f"{field.name}_id"
+            for field in self.model._meta.fields
+            if isinstance(field, models.ForeignKey)
+        ]
+        for field_name in ["run_id", "created_at", "created_by_id", "updated_at"]:
+            if field_name in field_names:
+                field_names.remove(field_name)
+                field_names.append(field_name)
+        if field_names[0] != "uid" and "uid" in field_names:
+            field_names.remove("uid")
+            field_names.insert(0, "uid")
+        # create the dataframe
+        df = pd.DataFrame(self.values(), columns=field_names)
         # if len(df) > 0 and "updated_at" in df:
         #     df.updated_at = format_and_convert_to_local_time(df.updated_at)
         # if len(df) > 0 and "started_at" in df:

lamindb/_registry.py

@@ -5,27 +5,21 @@ from typing import TYPE_CHECKING, Iterable, List, NamedTuple
 
 import dj_database_url
 import lamindb_setup as ln_setup
-from django.core.exceptions import FieldDoesNotExist
 from django.db import connections
-from django.db.models import Manager, Q, QuerySet
+from django.db.models import IntegerField, Manager, Q, QuerySet, Value
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
-from lamin_utils._search import search as base_search
 from lamindb_setup._connect_instance import get_owner_name_from_identifier
-from lamindb_setup._init_instance import InstanceSettings
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core._hub_core import connect_instance
-from lamindb_setup.core._settings_storage import StorageSettings
 from lnschema_core import Registry
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._settings import settings
-from lamindb.core.exceptions import ValidationError
 
 from ._from_values import get_or_create_records
 
 if TYPE_CHECKING:
-    import pandas as pd
     from lnschema_core.types import ListLike, StrField
 
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
@@ -53,31 +47,30 @@ def validate_required_fields(orm: Registry, kwargs):
         raise TypeError(f"{missing_fields} are required.")
 
 
-def suggest_objects_with_same_name(orm: Registry, kwargs) -> str | None:
+def suggest_records_with_similar_names(record: Registry, kwargs) -> bool:
+    """Returns True if found exact match, otherwise False.
+
+    Logs similar matches if found.
+    """
     if kwargs.get("name") is None:
-        return None
+        return False
+    queryset = _search(record.__class__, kwargs["name"], truncate_words=True, limit=5)
+    if not queryset.exists():  # empty queryset
+        return False
+    for alternative_record in queryset:
+        if alternative_record.name == kwargs["name"]:
+            return True
+    s, it, nots = ("", "it", "s") if len(queryset) == 1 else ("s", "one of them", "")
+    msg = f"record{s} with similar name{s} exist{nots}! did you mean to load {it}?"
+    if IPYTHON:
+        from IPython.display import display
+
+        logger.warning(f"{msg}")
+        if settings._verbosity_int >= 1:
+            display(queryset.df())
     else:
-        queryset = orm.search(kwargs["name"])
-        if not queryset.exists():  # empty queryset
-            return None
-        else:
-            for record in queryset:
-                if record.name == kwargs["name"]:
-                    return "object-with-same-name-exists"
-            else:
-                s, it = ("", "it") if len(queryset) == 1 else ("s", "one of them")
-                msg = (
-                    f"record{s} with similar name{s} exist! did you mean to load {it}?"
-                )
-                if IPYTHON:
-                    from IPython.display import display
-
-                    logger.warning(f"{msg}")
-                    if settings._verbosity_int >= 1:
-                        display(queryset.df())
-                else:
-                    logger.warning(f"{msg}\n{queryset}")
-    return None
+        logger.warning(f"{msg}\n{queryset}")
+    return False
 
 
 def __init__(orm: Registry, *args, **kwargs):
@@ -90,8 +83,8 @@ def __init__(orm: Registry, *args, **kwargs):
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
         if settings.upon_create_search_names and not has_consciously_provided_uid:
-            result = suggest_objects_with_same_name(orm, kwargs)
-            if result == "object-with-same-name-exists":
+            match = suggest_records_with_similar_names(orm, kwargs)
+            if match:
                 if "version" in kwargs:
                     version_comment = " and version"
                     existing_record = orm.filter(
@@ -101,10 +94,9 @@ def __init__(orm: Registry, *args, **kwargs):
                     version_comment = ""
                     existing_record = orm.filter(name=kwargs["name"]).one()
                 if existing_record is not None:
-                    logger.warning(
-                        f"loaded {orm.__class__.__name__} record with same"
-                        f" name{version_comment}: '{kwargs['name']}' "
-                        "(disable via `ln.settings.upon_create_search_names`)"
+                    logger.important(
+                        f"returning existing {orm.__class__.__name__} record with same"
+                        f" name{version_comment}: '{kwargs['name']}'"
                     )
                     init_self_from_db(orm, existing_record)
                     return None
@@ -123,6 +115,7 @@ def from_values(
     cls,
     values: ListLike,
     field: StrField | None = None,
+    create: bool = False,
     organism: Registry | str | None = None,
     public_source: Registry | None = None,
     mute: bool = False,
@@ -133,6 +126,7 @@ def from_values(
     return get_or_create_records(
         iterable=values,
         field=getattr(cls, field_str),
+        create=create,
         from_public=from_public,
         organism=organism,
         public_source=public_source,
@@ -156,6 +150,7 @@ def _search(
     limit: int | None = 20,
     case_sensitive: bool = False,
     using_key: str | None = None,
+    truncate_words: bool = False,
 ) -> QuerySet:
     input_queryset = _queryset(cls, using_key=using_key)
     orm = input_queryset.model
@@ -181,14 +176,49 @@ def _search(
                     ) from error
             else:
                 fields.append(field)
+
+    # decompose search string
+    def truncate_word(word) -> str:
+        if len(word) > 5:
+            n_80_pct = int(len(word) * 0.8)
+            return word[:n_80_pct]
+        elif len(word) > 3:
+            return word[:3]
+        else:
+            return word
+
+    decomposed_string = string.split()
+    # add the entire string back
+    decomposed_string += [string]
+    for word in decomposed_string:
+        # will not search against words with 3 or fewer characters
+        if len(word) <= 3:
+            decomposed_string.remove(word)
+    if truncate_words:
+        decomposed_string = [truncate_word(word) for word in decomposed_string]
+    # construct the query
     expression = Q()
     case_sensitive_i = "" if case_sensitive else "i"
     for field in fields:
-        # Construct the keyword for the Q object dynamically
+        for word in decomposed_string:
+            query = {f"{field}__{case_sensitive_i}contains": word}
+            expression |= Q(**query)
+    output_queryset = input_queryset.filter(expression)
+    # ensure exact matches are at the top
+    narrow_expression = Q()
+    for field in fields:
         query = {f"{field}__{case_sensitive_i}contains": string}
-        expression |= Q(**query)  # Unpack the dictionary into Q()
-    output_queryset = input_queryset.filter(expression)[:limit]
-    return output_queryset
+        narrow_expression |= Q(**query)
+    refined_output_queryset = output_queryset.filter(narrow_expression).annotate(
+        ordering=Value(1, output_field=IntegerField())
+    )
+    remaining_output_queryset = output_queryset.exclude(narrow_expression).annotate(
+        ordering=Value(2, output_field=IntegerField())
+    )
+    combined_queryset = refined_output_queryset.union(
+        remaining_output_queryset
+    ).order_by("ordering")[:limit]
+    return combined_queryset
 
 
 @classmethod  # type: ignore
@@ -376,11 +406,9 @@ def update_fk_to_default_db(
 FKBULK = [
     "organism",
     "public_source",
-    "initial_version",
     "latest_report",  # Transform
     "source_code",  # Transform
     "report",  # Run
-    "file",  # Collection
 ]
 
 
@@ -401,8 +429,8 @@ def transfer_to_default_db(
         registry = record.__class__
         record_on_default = registry.objects.filter(uid=record.uid).one_or_none()
         if record_on_default is not None:
-            logger.warning(
-                f"record with {record.uid} already exists on default database: {record}"
+            logger.important(
+                f"returning existing {record.__class__.__name__}(uid='{record.uid}') on default database"
             )
             return record_on_default
         if not mute:
@@ -481,9 +509,13 @@ def save(self, *args, **kwargs) -> Registry:
         if hasattr(self, "labels"):
             from copy import copy
 
+            from lnschema_core.models import FeatureManager
+
+            # here we go back to original record on the source database
             self_on_db = copy(self)
             self_on_db._state.db = db
-            self_on_db.pk = pk_on_db
+            self_on_db.pk = pk_on_db  # manually set the primary key
+            self_on_db.features = FeatureManager(self_on_db)
             # by default, transfer parents of the labels to maintain ontological hierarchy
             try:
                 import bionty as bt
@@ -492,9 +524,7 @@ def save(self, *args, **kwargs) -> Registry:
             except ImportError:
                 parents = kwargs.get("parents", True)
             add_from_kwargs = {"parents": parents}
-            logger.info("transfer features")
             self.features._add_from(self_on_db, **add_from_kwargs)
-            logger.info("transfer labels")
             self.labels.add_from(self_on_db, **add_from_kwargs)
     return self
 
@@ -519,24 +549,3 @@ if ln_setup._TESTING:  # type: ignore
 
 for name in METHOD_NAMES:
     attach_func_to_class_method(name, Registry, globals())
-
-
-@classmethod  # type: ignore
-def __get_schema_name__(cls) -> str:
-    schema_module_name = cls.__module__.split(".")[0]
-    schema_name = schema_module_name.replace("lnschema_", "")
-    return schema_name
-
-
-@classmethod  # type: ignore
-def __get_name_with_schema__(cls) -> str:
-    schema_name = cls.__get_schema_name__()
-    if schema_name == "core":
-        schema_prefix = ""
-    else:
-        schema_prefix = f"{schema_name}."
-    return f"{schema_prefix}{cls.__name__}"
-
-
-Registry.__get_schema_name__ = __get_schema_name__
-Registry.__get_name_with_schema__ = __get_name_with_schema__

lamindb/_save.py

@@ -148,13 +148,17 @@ def check_and_attempt_upload(
     artifact: Artifact,
     using_key: str | None = None,
     access_token: str | None = None,
+    print_progress: bool = True,
 ) -> Exception | None:
     # if Artifact object is either newly instantiated or replace() was called on
     # a local env it will have a _local_filepath and needs to be uploaded
     if hasattr(artifact, "_local_filepath"):
         try:
             storage_path = upload_artifact(
-                artifact, using_key, access_token=access_token
+                artifact,
+                using_key,
+                access_token=access_token,
+                print_progress=print_progress,
             )
         except Exception as exception:
             logger.warning(f"could not upload artifact: {artifact}")
@@ -264,6 +268,8 @@ def store_artifacts(
                 if artifact not in stored_artifacts:
                     artifact._delete_skip_storage()
         error_message = prepare_error_message(artifacts, stored_artifacts, exception)
+        # this is bad because we're losing the original traceback
+        # needs to be refactored - also, the orginal error should be raised
         raise RuntimeError(error_message)
     return None
 
@@ -289,7 +295,10 @@ def prepare_error_message(records, stored_artifacts, exception) -> str:
 
 
 def upload_artifact(
-    artifact, using_key: str | None = None, access_token: str | None = None
+    artifact,
+    using_key: str | None = None,
+    access_token: str | None = None,
+    print_progress: bool = True,
 ) -> UPath:
     """Store and add file and its linked entries."""
     # can't currently use  filepath_from_artifact here because it resolves to ._local_filepath
@@ -299,5 +308,7 @@ def upload_artifact(
     )
     if hasattr(artifact, "_to_store") and artifact._to_store:
         logger.save(f"storing artifact '{artifact.uid}' at '{storage_path}'")
-        store_file_or_folder(artifact._local_filepath, storage_path)
+        store_file_or_folder(
+            artifact._local_filepath, storage_path, print_progress=print_progress
+        )
     return storage_path

lamindb/_ulabel.py

@@ -8,8 +8,6 @@ from lnschema_core import ULabel
 
 from lamindb._utils import attach_func_to_class_method
 
-from ._from_values import get_or_create_records
-
 if TYPE_CHECKING:
     from lnschema_core.types import ListLike
 
@@ -41,20 +39,8 @@ def __init__(self, *args, **kwargs):
     )
 
 
-@classmethod  # type:ignore
-@doc_args(ULabel.from_values.__doc__)
-def from_values(cls, values: ListLike, **kwargs) -> list[ULabel]:
-    """{}."""
-    records = get_or_create_records(
-        iterable=values,
-        field=ULabel.name,
-    )
-    return records
-
-
 METHOD_NAMES = [
     "__init__",
-    "from_values",
 ]
 
 if ln_setup._TESTING:

lamindb/core/__init__.py

@@ -9,7 +9,7 @@ Registries:
    QuerySet
    QueryManager
    RecordsList
-   Data
+   HasFeatures
    FeatureManager
    LabelManager
    IsVersioned
@@ -36,6 +36,7 @@ Classes:
    :toctree: .
 
    Settings
+   TransformSettings
    MappedCollection
    run_context
 
@@ -54,7 +55,7 @@ Modules:
 from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
-    Data,
+    HasFeatures,
     HasParents,
     IsVersioned,
     Registry,
@@ -77,3 +78,4 @@ from . import _data, datasets, exceptions, fields, types
 from ._mapped_collection import MappedCollection
 from ._run_context import run_context
 from ._settings import Settings
+from ._transform_settings import TransformSettings

lamindb/core/_data.py

@@ -8,9 +8,9 @@ from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import (
     Artifact,
     Collection,
-    Data,
     Feature,
     FeatureSet,
+    HasFeatures,
     Registry,
     Run,
     ULabel,
@@ -20,6 +20,7 @@ from lnschema_core.models import (
 
 from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
+from lamindb._registry import get_default_str_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
@@ -94,25 +95,8 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
         bulk_create(links, ignore_conflicts=True)
 
 
-def format_repr(value: Registry, exclude: list[str] | str | None = None) -> str:
-    if isinstance(exclude, str):
-        exclude = [exclude]
-    exclude_fields = set() if exclude is None else set(exclude)
-    exclude_fields.update(["created_at", "updated_at"])
-
-    fields = [
-        f
-        for f in value.__repr__(include_foreign_keys=False).split(", ")
-        if not any(f"{excluded_field}=" in f for excluded_field in exclude_fields)
-    ]
-    repr = ", ".join(fields)
-    if not repr.endswith(")"):
-        repr += ")"
-    return repr
-
-
-@doc_args(Data.describe.__doc__)
-def describe(self: Data):
+@doc_args(HasFeatures.describe.__doc__)
+def describe(self: HasFeatures, print_types: bool = False):
     """{}."""
     # prefetch all many-to-many relationships
     # doesn't work for describing using artifact
@@ -125,7 +109,8 @@ def describe(self: Data):
     # )
 
     model_name = self.__class__.__name__
-    msg = ""
+    msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}\n"
+    prov_msg = ""
 
     fields = self._meta.fields
     direct_fields = []
@@ -151,26 +136,30 @@ def describe(self: Data):
 
     # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
-        record_msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}"
-        msg += f"{record_msg}\n\n"
-
-        msg += f"{colors.green('Provenance')}:\n  "
+        fields_values = [(field, getattr(self, field)) for field in foreign_key_fields]
+        type_str = (
+            lambda attr: f": {attr.__class__.__get_name_with_schema__()}"
+            if print_types
+            else ""
+        )
         related_msg = "".join(
             [
-                f"📎 {field}: {format_repr(self.__getattribute__(field))}\n  "
-                for field in foreign_key_fields
-                if self.__getattribute__(field) is not None
+                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_default_str_field(attr)))}\n"
+                for (field_name, attr) in fields_values
+                if attr is not None
             ]
         )
-        msg += related_msg
+        prov_msg += related_msg
     # input of
     if self.id is not None and self.input_of.exists():
         values = [format_field_value(i.started_at) for i in self.input_of.all()]
-        msg += f"📎 input_of ({colors.italic('core.Run')}): {values}\n    "
-    msg = msg.rstrip(" ")  # do not use removesuffix as we need to remove 2 or 4 spaces
-    msg += print_features(self)
-    msg += print_labels(self)
-
+        type_str = ": Run" if print_types else ""  # type: ignore
+        prov_msg += f"    .input_of{type_str} = {values}\n"
+    if prov_msg:
+        msg += f"  {colors.italic('Provenance')}\n"
+        msg += prov_msg
+    msg += print_labels(self, print_types=print_types)
+    msg += print_features(self, print_types=print_types)  # type: ignore
     logger.print(msg)
 
 
@@ -212,9 +201,9 @@ def get_labels(
             qs_by_registry[registry] = ULabel.objects.using(self._state.db).filter(
                 id__in=label_ids
             )
-        elif registry in self.features.accessor_by_orm:
+        elif registry in self.features._accessor_by_registry:
             qs_by_registry[registry] = getattr(
-                self, self.features.accessor_by_orm[registry]
+                self, self.features._accessor_by_registry[registry]
             ).all()
     if flat_names:
         # returns a flat list of names
@@ -301,11 +290,11 @@ def add_labels(
                 record
             )
         for registry_name, records in records_by_registry.items():
-            if registry_name not in self.features.accessor_by_orm:
+            if registry_name not in self.features._accessor_by_registry:
                 logger.warning(f"skipping {registry_name}")
                 continue
             labels_accessor = getattr(
-                self, self.features.accessor_by_orm[registry_name]
+                self, self.features._accessor_by_registry[registry_name]
             )
             # remove labels that are already linked as add doesn't perform update
             linked_labels = [r for r in records if r in labels_accessor.filter()]
@@ -316,7 +305,7 @@ def add_labels(
         feature_set_ids = [link.featureset_id for link in feature_set_links.all()]
         # get all linked features of type Feature
         feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
-        linked_features_by_slot = {
+        {
             feature_set_links.filter(featureset_id=feature_set.id)
             .one()
             .slot: feature_set.features.all()
@@ -324,62 +313,19 @@ def add_labels(
             if "Feature" == feature_set.registry
         }
         for registry_name, _ in records_by_registry.items():
-            msg = ""
-            if (
-                not feature.dtype.startswith("cat[")
-                or registry_name not in feature.dtype
-            ):
-                if len(msg) > 0:
-                    msg += ", "
-                msg += f"linked feature '{feature.name}' to registry '{registry_name}'"
+            if registry_name not in feature.dtype:
+                logger.debug(
+                    f"updated categorical feature '{feature.name}' type with registry '{registry_name}'"
+                )
                 if not feature.dtype.startswith("cat["):
                     feature.dtype = f"cat[{registry_name}]"
                 elif registry_name not in feature.dtype:
                     feature.dtype = feature.dtype.rstrip("]") + f"|{registry_name}]"
                 feature.save()
-            if len(msg) > 0:
-                logger.save(msg)
-            # check whether we have to update the feature set that manages labels
-            # (Feature) to account for a new feature
-            found_feature = False
-            for _, linked_features in linked_features_by_slot.items():
-                if feature in linked_features:
-                    found_feature = True
-            if not found_feature:
-                if "external" in linked_features_by_slot:
-                    feature_set = self.features.feature_set_by_slot["external"]
-                    features_list = feature_set.features.list()
-                else:
-                    features_list = []
-                features_list.append(feature)
-                feature_set = FeatureSet(features_list)
-                feature_set.save()
-                if "external" in linked_features_by_slot:
-                    old_feature_set_link = feature_set_links.filter(
-                        slot="external"
-                    ).one()
-                    old_feature_set_link.delete()
-                    remaining_links = self.feature_sets.through.objects.filter(
-                        featureset_id=feature_set.id
-                    ).all()
-                    if len(remaining_links) == 0:
-                        old_feature_set = FeatureSet.filter(
-                            id=old_feature_set_link.featureset_id
-                        ).one()
-                        logger.info(
-                            "nothing links to it anymore, deleting feature set"
-                            f" {old_feature_set}"
-                        )
-                        old_feature_set.delete()
-                self.features.add_feature_set(feature_set, slot="external")
-                logger.save(
-                    f"linked new feature '{feature.name}' together with new feature set"
-                    f" {feature_set}"
-                )
 
 
 def _track_run_input(
-    data: Data | Iterable[Data],
+    data: HasFeatures | Iterable[HasFeatures],
     is_run_input: bool | None = None,
     run: Run | None = None,
 ):
@@ -391,12 +337,12 @@ def _track_run_input(
     elif run is None:
         run = run_context.run
     # consider that data is an iterable of Data
-    data_iter: Iterable[Data] = [data] if isinstance(data, Data) else data
+    data_iter: Iterable[HasFeatures] = [data] if isinstance(data, HasFeatures) else data
     track_run_input = False
     input_data = []
     if run is not None:
         # avoid cycles: data can't be both input and output
-        def is_valid_input(data: Data):
+        def is_valid_input(data: HasFeatures):
             return (
                 data.run_id != run.id
                 and not data._state.adding
@@ -432,7 +378,7 @@ def _track_run_input(
                                 f" {input_data[0].transform.id}"
                             )
                     logger.info(
-                        f"adding {data_class_name} {input_data_ids} as input for run"
+                        f"adding {data_class_name} ids {input_data_ids} as inputs for run"
                         f" {run.id}{transform_note}"
                     )
                     track_run_input = True
@@ -469,25 +415,5 @@ def _track_run_input(
                 run.transform.parents.add(input_data[0].transform)
 
 
-@property  # type: ignore
-@doc_args(Data.features.__doc__)
-def features(self) -> FeatureManager:
-    """{}."""
-    from lamindb.core._feature_manager import FeatureManager
-
-    return FeatureManager(self)
-
-
-@property  # type: ignore
-@doc_args(Data.labels.__doc__)
-def labels(self) -> LabelManager:
-    """{}."""
-    from lamindb.core._label_manager import LabelManager
-
-    return LabelManager(self)
-
-
-Data.features = features
-Data.labels = labels
-Data.describe = describe
-Data.view_lineage = view_lineage
+HasFeatures.describe = describe
+HasFeatures.view_lineage = view_lineage