lamindb 0.76.12__py3-none-any.whl → 0.76.14__py3-none-any.whl

lamindb/_feature_set.py

@@ -216,7 +216,9 @@ def members(self) -> QuerySet:
 
 
 def _get_related_name(self: FeatureSet) -> str:
-    feature_sets_related_models = dict_related_model_to_related_name(self)
+    feature_sets_related_models = dict_related_model_to_related_name(
+        self, instance=self._state.db
+    )
     related_name = feature_sets_related_models.get(self.registry)
     return related_name
 

lamindb/_from_values.py

@@ -64,11 +64,7 @@ def get_or_create_records(
             if source_record:
                 from bionty.core._add_ontology import check_source_in_db
 
-                check_source_in_db(
-                    registry=registry,
-                    source=source_record,
-                    update=True,
-                )
+                check_source_in_db(registry=registry, source=source_record)
 
                 from_source = not source_record.in_db
             elif hasattr(registry, "source_id"):

lamindb/_parents.py

@@ -310,7 +310,12 @@ def _record_label(record: Record, field: str | None = None):
             rf' FACE="Monospace">uid={record.uid}<BR/>version={record.version}</FONT>>'
         )
     elif isinstance(record, Run):
-        name = f'{record.transform.name.replace("&", "&amp;")}'
+        if record.transform.name:
+            name = f'{record.transform.name.replace("&", "&amp;")}'
+        elif record.transform.key:
+            name = f'{record.transform.key.replace("&", "&amp;")}'
+        else:
+            name = f"{record.transform.uid}"
         user_display = (
             record.created_by.handle
             if record.created_by.name is None
@@ -365,7 +370,6 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
                 inputs_run += (
                     r.input_collections.all().filter(visibility__in=[0, 1]).list()
                 )
-            run_inputs_outputs += [(inputs_run, r)]
             outputs_run = (
                 r.__getattribute__(f"output_{name}s")
                 .all()
@@ -376,7 +380,18 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
                 outputs_run += (
                     r.output_collections.all().filter(visibility__in=[0, 1]).list()
                 )
-            run_inputs_outputs += [(r, outputs_run)]
+            # if inputs are outputs artifacts are the same, will result infinite loop
+            # so only show as outputs
+            overlap = set(inputs_run).intersection(outputs_run)
+            if overlap:
+                logger.warning(
+                    f"The following artifacts are both inputs and outputs of Run(uid={r.uid}): {overlap}\n   → Only showing as outputs."
+                )
+                inputs_run = list(set(inputs_run) - overlap)
+            if len(inputs_run) > 0:
+                run_inputs_outputs += [(inputs_run, r)]
+            if len(outputs_run) > 0:
+                run_inputs_outputs += [(r, outputs_run)]
             inputs += inputs_run
         runs = [f.run for f in inputs if f.run is not None]
     return run_inputs_outputs

lamindb/_query_manager.py

@@ -98,26 +98,11 @@ class QueryManager(models.Manager):
 
         return _lookup(cls=self.all(), field=field, **kwargs)
 
-    def __getitem__(self, item: str):
-        try:
-            source_field_name = self.source_field_name
-            target_field_name = self.target_field_name
-
-            if (
-                source_field_name in {"artifact", "collection"}
-                and target_field_name == "feature_set"
-            ):
-                return get_feature_set_by_slot_(host=self.instance).get(item)
-
-        except Exception:  # pragma: no cover
-            return
-
 
 models.Manager.list = QueryManager.list
 models.Manager.df = QueryManager.df
 models.Manager.search = QueryManager.search
 models.Manager.lookup = QueryManager.lookup
-models.Manager.__getitem__ = QueryManager.__getitem__
 models.Manager._track_run_input_manager = QueryManager._track_run_input_manager
 # the two lines below would be easy if we could actually inherit; like this,
 # they're suboptimal

lamindb/_query_set.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, NamedTuple
 import pandas as pd
 from django.db import models
 from django.db.models import F
-from lamin_utils import logger
+from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import (
     Artifact,
@@ -115,7 +115,7 @@ def get(
     else:
         assert idlike is None  # noqa: S101
         expressions = process_expressions(registry, expressions)
-        return registry.objects.get(**expressions)
+        return registry.objects.using(qs.db).get(**expressions)
 
 
 class RecordsList(UserList):
@@ -186,6 +186,7 @@ class QuerySet(models.QuerySet):
         if pk_column_name in df.columns:
             df = df.set_index(pk_column_name)
         if len(df) == 0:
+            logger.warning(colors.yellow("No records found"))
             return df
         if include is not None:
             if isinstance(include, str):
@@ -218,12 +219,15 @@ class QuerySet(models.QuerySet):
                             f"{related_ORM.__name__.lower()}__{lookup_str}"
                         )
                     link_df = pd.DataFrame(
-                        field.through.objects.values(
+                        field.through.objects.using(self.db).values(
                             left_side_link_model, values_expression
                         )
                     )
                     if link_df.shape[0] == 0:
-                        return df
+                        logger.warning(
+                            f"{colors.yellow(expression)} is not shown because no values are found"
+                        )
+                        continue
                     link_groupby = link_df.groupby(left_side_link_model)[
                         values_expression
                     ].apply(list)

lamindb/_record.py

@@ -7,7 +7,7 @@ import dj_database_url
 import lamindb_setup as ln_setup
 from django.db import connections, transaction
 from django.db.models import IntegerField, Manager, Q, QuerySet, Value
-from lamin_utils import logger
+from lamin_utils import colors, logger
 from lamin_utils._lookup import Lookup
 from lamindb_setup._connect_instance import (
     get_owner_name_from_identifier,
@@ -17,10 +17,11 @@ from lamindb_setup._connect_instance import (
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core._hub_core import connect_instance_hub
 from lamindb_setup.core._settings_store import instance_settings_file
-from lnschema_core.models import IsVersioned, Record, Run, Transform
+from lnschema_core.models import Artifact, Feature, IsVersioned, Record, Run, Transform
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._settings import settings
+from lamindb.core.exceptions import RecordNameChangeIntegrityError
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -129,6 +130,7 @@ def __init__(record: Record, *args, **kwargs):
     else:
         # object is loaded from DB (**kwargs could be omitted below, I believe)
         super(Record, record).__init__(*args, **kwargs)
+        _store_record_old_name(record)
 
 
 @classmethod  # type:ignore
@@ -376,6 +378,8 @@ def using(
     instance: str | None,
 ) -> QuerySet:
     """{}"""  # noqa: D415
+    from ._query_set import QuerySet
+
     if instance is None:
         return QuerySet(model=cls, using=None)
     owner, name = get_owner_name_from_identifier(instance)
@@ -395,15 +399,15 @@ def using(
         if not source_schema.issubset(target_schema):
             missing_members = source_schema - target_schema
             logger.warning(
-                f"source schema has additional modules: {missing_members}\nconsider mounting these schema modules to not encounter errors"
+                f"source schema has additional modules: {missing_members}\nconsider mounting these schema modules to transfer all metadata"
             )
-        cache_filepath.write_text(iresult["lnid"])  # type: ignore
+        cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}")  # type: ignore
         settings_file = instance_settings_file(name, owner)
         db = update_db_using_local(iresult, settings_file)
     else:
         isettings = load_instance_settings(settings_file)
         db = isettings.db
-        cache_filepath.write_text(isettings.uid)
+        cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.schema)}")  # type: ignore
     add_db_connection(db, instance)
     return QuerySet(model=cls, using=instance)
 
@@ -470,7 +474,7 @@ def get_transfer_run(record) -> Run:
     cache_filepath = ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
     if not cache_filepath.exists():
         raise SystemExit("Need to call .using() before")
-    instance_uid = cache_filepath.read_text()
+    instance_uid = cache_filepath.read_text().split("\n")[0]
     key = f"transfers/{instance_uid}"
     uid = instance_uid + "0000"
     transform = Transform.filter(uid=uid).one_or_none()
@@ -582,11 +586,15 @@ def save(self, *args, **kwargs) -> Record:
             with transaction.atomic():
                 revises._revises = None  # ensure we don't start a recursion
                 revises.save()
+                check_name_change(self)
                 super(Record, self).save(*args, **kwargs)
+                _store_record_old_name(self)
             self._revises = None
         # save unversioned record
         else:
+            check_name_change(self)
             super(Record, self).save(*args, **kwargs)
+            _store_record_old_name(self)
     # perform transfer of many-to-many fields
     # only supported for Artifact and Collection records
     if db is not None and db != "default" and using_key is None:
@@ -614,6 +622,74 @@ def save(self, *args, **kwargs) -> Record:
     return self
 
 
+def _store_record_old_name(record: Record):
+    # writes the name to the _name attribute, so we can detect renaming upon save
+    if hasattr(record, "_name_field"):
+        record._name = getattr(record, record._name_field)
+
+
+def check_name_change(record: Record):
+    """Warns if a record's name has changed."""
+    if (
+        not record.pk
+        or not hasattr(record, "_name")
+        or not hasattr(record, "_name_field")
+    ):
+        return
+
+    old_name = record._name
+    new_name = getattr(record, record._name_field)
+    registry = record.__class__.__name__
+
+    if old_name != new_name:
+        # when a label is renamed, only raise a warning if it has a feature
+        if hasattr(record, "artifacts"):
+            linked_records = (
+                record.artifacts.through.filter(
+                    label_ref_is_name=True, **{f"{registry.lower()}_id": record.pk}
+                )
+                .exclude(feature_id=None)  # must have a feature
+                .exclude(
+                    feature_ref_is_name=None
+                )  # must be linked via Curator and therefore part of a featureset
+                .distinct()
+            )
+            artifact_ids = linked_records.list("artifact__uid")
+            n = len(artifact_ids)
+            s = "s" if n > 1 else ""
+            if n > 0:
+                logger.error(
+                    f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
+                    f"   → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
+                    f"{colors.bold('To rename this label')}, make it external:\n"
+                    f"   → run `artifact.labels.make_external(label)`\n\n"
+                    f"After renaming, consider re-curating the above artifact{s}:\n"
+                    f'   → in each dataset, manually modify label "{old_name}" to "{new_name}"\n'
+                    f"   → run `ln.Curator`\n"
+                )
+                raise RecordNameChangeIntegrityError
+
+        # when a feature is renamed
+        elif isinstance(record, Feature):
+            # only internal features are associated with featuresets
+            linked_artifacts = Artifact.filter(feature_sets__features=record).list(
+                "uid"
+            )
+            n = len(linked_artifacts)
+            s = "s" if n > 1 else ""
+            if n > 0:
+                logger.error(
+                    f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
+                    f"   → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
+                    f"{colors.bold('To rename this feature')}, make it external:\n"
+                    "   → run `artifact.features.make_external(feature)`\n\n"
+                    f"After renaming, consider re-curating the above artifact{s}:\n"
+                    f"   → in each dataset, manually modify feature '{old_name}' to '{new_name}'\n"
+                    f"   → run `ln.Curator`\n"
+                )
+                raise RecordNameChangeIntegrityError
+
+
 def delete(self) -> None:
     """Delete the record."""
     # note that the logic below does not fire if a record is moved to the trash

lamindb/core/__init__.py

@@ -60,9 +60,11 @@ Modules:
    types
    exceptions
    subsettings
+   logger
 
 """
 
+from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,

lamindb/core/_context.py

@@ -438,7 +438,7 @@ class Context:
             )
             return (
                 f'Filename "{key}" clashes with the existing key "{transform.key}" for uid "{transform.uid[:-4]}...."\n\nEither init a new transform with a new uid:\n\n'
-                f'ln.track("{ids.base62_12()}0000)"\n\n{update_key_note}'
+                f'ln.track("{ids.base62_12()}0000")\n\n{update_key_note}'
             )
 
         # make a new transform record

lamindb/core/_data.py

@@ -26,6 +26,7 @@ from lamindb.core._settings import settings
 from ._context import context
 from ._django import get_artifact_with_related, get_related_model
 from ._feature_manager import (
+    add_label_feature_links,
     get_feature_set_links,
     get_host_id_field,
     get_label_links,
@@ -67,11 +68,17 @@ def add_transform_to_kwargs(kwargs: dict[str, Any], run: Run):
 
 def save_feature_sets(self: Artifact | Collection) -> None:
     if hasattr(self, "_feature_sets"):
+        from lamindb.core._feature_manager import get_feature_set_by_slot_
+
+        existing_feature_sets = get_feature_set_by_slot_(self)
         saved_feature_sets = {}
         for key, feature_set in self._feature_sets.items():
             if isinstance(feature_set, FeatureSet) and feature_set._state.adding:
                 feature_set.save()
                 saved_feature_sets[key] = feature_set
+            if key in existing_feature_sets:
+                # remove existing feature set on the same slot
+                self.feature_sets.remove(existing_feature_sets[key])
         if len(saved_feature_sets) > 0:
             s = "s" if len(saved_feature_sets) > 1 else ""
             display_feature_set_keys = ",".join(
@@ -305,6 +312,8 @@ def add_labels(
     feature: Feature | None = None,
     *,
     field: StrField | None = None,
+    feature_ref_is_name: bool | None = None,
+    label_ref_is_name: bool | None = None,
 ) -> None:
     """{}"""  # noqa: D415
     if self._state.adding:
@@ -373,14 +382,17 @@ def add_labels(
             if registry_name not in self.features._accessor_by_registry:
                 logger.warning(f"skipping {registry_name}")
                 continue
-            labels_accessor = getattr(
-                self, self.features._accessor_by_registry[registry_name]
+            if len(records) == 0:
+                continue
+            features_labels = {
+                registry_name: [(feature, label_record) for label_record in records]
+            }
+            add_label_feature_links(
+                self.features,
+                features_labels,
+                feature_ref_is_name=feature_ref_is_name,
+                label_ref_is_name=label_ref_is_name,
             )
-            # remove labels that are already linked as add doesn't perform update
-            linked_labels = [r for r in records if r in labels_accessor.filter()]
-            if len(linked_labels) > 0:
-                labels_accessor.remove(*linked_labels)
-            labels_accessor.add(*records, through_defaults={"feature_id": feature.id})
         links_feature_set = get_feature_set_links(self)
         feature_set_ids = [link.featureset_id for link in links_feature_set.all()]
         # get all linked features of type Feature

lamindb/core/_django.py

@@ -6,7 +6,7 @@ from django.db.models.fields.reverse_related import ManyToManyRel, ManyToOneRel
 from django.db.models.functions import JSONObject
 from lnschema_core.models import Artifact, FeatureSet, Record
 
-from .schema import dict_related_model_to_related_name
+from .schema import dict_related_model_to_related_name, get_schemas_modules
 
 
 def get_related_model(model, field_name):
@@ -35,22 +35,36 @@ def get_artifact_with_related(
     """Fetch an artifact with its related data."""
     from lamindb._can_validate import get_name_field
 
+    from ._label_manager import LABELS_EXCLUDE_SET
+
     model = artifact.__class__
-    foreign_key_fields = [f.name for f in model._meta.fields if f.is_relation]
+    schema_modules = get_schemas_modules(artifact._state.db)
+
+    foreign_key_fields = [
+        f.name
+        for f in model._meta.fields
+        if f.is_relation and f.related_model.__get_schema_name__() in schema_modules
+    ]
 
     m2m_relations = (
         []
         if not include_m2m
         else [
             v
-            for v in dict_related_model_to_related_name(model).values()
-            if not v.startswith("_")
+            for v in dict_related_model_to_related_name(
+                model, instance=artifact._state.db
+            ).values()
+            if not v.startswith("_") and v not in LABELS_EXCLUDE_SET
         ]
     )
     link_tables = (
         []
         if not include_feature_link
-        else list(dict_related_model_to_related_name(model, links=True).values())
+        else list(
+            dict_related_model_to_related_name(
+                model, links=True, instance=artifact._state.db
+            ).values()
+        )
     )
 
     # Clear previous queries

lamindb/core/_feature_manager.py

@@ -13,6 +13,7 @@ from django.contrib.postgres.aggregates import ArrayAgg
 from django.db import connections
 from django.db.models import Aggregate
 from lamin_utils import colors, logger
+from lamindb_setup.core.hashing import hash_set
 from lamindb_setup.core.upath import create_path
 from lnschema_core.models import (
     Artifact,
@@ -216,9 +217,11 @@ def _print_categoricals(
     if not print_params:
         labels_msg = ""
         labels_by_feature = defaultdict(list)
-        for _, (_, links) in get_labels_as_dict(self, links=True).items():
+        for _, (_, links) in get_labels_as_dict(
+            self, links=True, instance=self._state.db
+        ).items():
             for link in links:
-                if link.feature_id is not None:
+                if hasattr(link, "feature_id") and link.feature_id is not None:
                     link_attr = get_link_attr(link, self)
                     labels_by_feature[link.feature_id].append(
                         getattr(link, link_attr).name
@@ -584,6 +587,48 @@ def _accessor_by_registry(self):
     return self._accessor_by_registry_
 
 
+def add_label_feature_links(
+    self,
+    features_labels,
+    *,
+    label_ref_is_name: bool | None = None,
+    feature_ref_is_name: bool | None = None,
+):
+    if list(features_labels.keys()) != ["ULabel"]:
+        related_names = dict_related_model_to_related_name(self._host.__class__)
+    else:
+        related_names = {"ULabel": "ulabels"}
+    for class_name, registry_features_labels in features_labels.items():
+        related_name = related_names[class_name]  # e.g., "ulabels"
+        LinkORM = getattr(self._host, related_name).through
+        field_name = f"{get_link_attr(LinkORM, self._host)}_id"  # e.g., ulabel_id
+        links = [
+            LinkORM(
+                **{
+                    "artifact_id": self._host.id,
+                    "feature_id": feature.id,
+                    field_name: label.id,
+                    "feature_ref_is_name": feature_ref_is_name,
+                    "label_ref_is_name": label_ref_is_name,
+                }
+            )
+            for (feature, label) in registry_features_labels
+        ]
+        # a link might already exist
+        try:
+            save(links, ignore_conflicts=False)
+        except Exception:
+            save(links, ignore_conflicts=True)
+        # now delete links that were previously saved without a feature
+        LinkORM.filter(
+            **{
+                "artifact_id": self._host.id,
+                "feature_id": None,
+                f"{field_name}__in": [l.id for _, l in registry_features_labels],
+            }
+        ).all().delete()
+
+
 def _add_values(
     self,
     values: dict[str, str | int | float | bool],
@@ -715,49 +760,9 @@ def _add_values(
             f"Here is how to create ulabels for them:\n\n{hint}"
         )
         raise ValidationError(msg)
-    # bulk add all links to ArtifactULabel
+    # bulk add all links
     if features_labels:
-        if list(features_labels.keys()) != ["ULabel"]:
-            related_names = dict_related_model_to_related_name(self._host.__class__)
-        else:
-            related_names = {"ULabel": "ulabels"}
-        for class_name, registry_features_labels in features_labels.items():
-            related_name = related_names[class_name]  # e.g., "ulabels"
-            LinkORM = getattr(self._host, related_name).through
-            field_name = f"{get_link_attr(LinkORM, self._host)}_id"  # e.g., ulabel_id
-            links = [
-                LinkORM(
-                    **{
-                        "artifact_id": self._host.id,
-                        "feature_id": feature.id,
-                        field_name: label.id,
-                    }
-                )
-                for (feature, label) in registry_features_labels
-            ]
-            # a link might already exist
-            try:
-                save(links, ignore_conflicts=False)
-            except Exception:
-                save(links, ignore_conflicts=True)
-                # now deal with links that were previously saved without a feature_id
-                links_saved = LinkORM.filter(
-                    **{
-                        "artifact_id": self._host.id,
-                        f"{field_name}__in": [
-                            l.id for _, l in registry_features_labels
-                        ],
-                    }
-                )
-                for link in links_saved.all():
-                    # TODO: also check for inconsistent features
-                    if link.feature_id is None:
-                        link.feature_id = [
-                            f.id
-                            for f, l in registry_features_labels
-                            if l.id == getattr(link, field_name)
-                        ][0]
-                        link.save()
+        add_label_feature_links(self, features_labels)
     if _feature_values:
         save(_feature_values)
         if is_param:
@@ -1004,6 +1009,36 @@ def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
         self._host.features.add_feature_set(feature_set_self, slot)
 
 
+def make_external(self, feature: Feature) -> None:
+    """Make a feature external, aka, remove feature from feature sets.
+
+    Args:
+        feature: `Feature` A feature record.
+
+    """
+    if not isinstance(feature, Feature):
+        raise TypeError("feature must be a Feature record!")
+    feature_sets = FeatureSet.filter(features=feature).all()
+    for fs in feature_sets:
+        f = Feature.filter(uid=feature.uid).all()
+        features_updated = fs.members.difference(f)
+        if len(features_updated) > 0:
+            # re-compute the hash of feature sets based on the updated members
+            features_hash = hash_set({feature.uid for feature in features_updated})
+            fs.hash = features_hash
+            fs.n = len(features_updated)
+            fs.save()
+        # delete the link between the feature and the feature set
+        FeatureSet.features.through.objects.filter(
+            feature_id=feature.id, featureset_id=fs.id
+        ).delete()
+        # if no members are left in the featureset, delete it
+        if len(features_updated) == 0:
+            logger.warning(f"deleting empty feature set: {fs}")
+            fs.artifacts.set([])
+            fs.delete()
+
+
 FeatureManager.__init__ = __init__
 ParamManager.__init__ = __init__
 FeatureManager.__repr__ = __repr__
@@ -1020,6 +1055,7 @@ FeatureManager._add_set_from_mudata = _add_set_from_mudata
 FeatureManager._add_from = _add_from
 FeatureManager.filter = filter
 FeatureManager.get = get
+FeatureManager.make_external = make_external
 ParamManager.add_values = add_values_params
 ParamManager.get_values = get_values
 ParamManager.filter = filter