lamindb 0.75.0__py3-none-any.whl → 0.76.0__py3-none-any.whl

lamindb/_feature.py

@@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs):
         return None
     # now we proceed with the user-facing constructor
     if len(args) != 0:
-        raise ValueError("Only non-keyword args allowed")
+        raise ValueError("Only keyword args allowed")
     dtype: type | str = kwargs.pop("dtype") if "dtype" in kwargs else None
     # cast type
     if dtype is None:

lamindb/_from_values.py

@@ -47,15 +47,35 @@ def get_or_create_records(
 
         # new records to be created based on new values
         if len(nonexist_values) > 0:
-            if source:
-                from_source = not source.in_db
-            elif (
-                records
-                and hasattr(records[0], "source_id")
-                and records[0].source_id
-                and records[0].source.in_db
-            ):
+            source_record = None
+            if from_source:
+                if isinstance(source, Record):
+                    source_record = source
+                elif (
+                    len(records) > 0
+                    and hasattr(records[0], "source_id")
+                    and records[0].source_id
+                ):
+                    source_record = records[0].source
+            if not source_record and hasattr(Record, "public"):
+                from bionty._bionty import get_source_record
+
+                source_record = get_source_record(Record.public(organism=organism))
+            if source_record:
+                from bionty.core._add_ontology import check_source_in_db
+
+                check_source_in_db(
+                    registry=Record,
+                    source=source_record,
+                    update=True,
+                )
+
+                from_source = not source_record.in_db
+            elif hasattr(Record, "source_id"):
+                from_source = True
+            else:
                 from_source = False
+
             if from_source:
                 records_bionty, unmapped_values = create_records_from_source(
                     iterable_idx=nonexist_values,
@@ -211,10 +231,6 @@ def create_records_from_source(
         return records, iterable_idx
     # add source record to the kwargs
     source_record = get_source_record(public_ontology)
-    if source_record is not None and source_record.in_db:
-        # skips the creation of records from public if the source is already in the db
-        return records, iterable_idx
-
     kwargs.update({"source": source_record})
 
     # filter the columns in bionty df based on fields
@@ -335,9 +351,9 @@ def _bulk_create_dicts_from_df(
     return df.reset_index().to_dict(orient="records"), multi_msg
 
 
-def _has_organism_field(orm: type[Record]) -> bool:
+def _has_organism_field(registry: type[Record]) -> bool:
     try:
-        orm._meta.get_field("organism")
+        registry._meta.get_field("organism")
         return True
     except FieldDoesNotExist:
         return False
@@ -346,17 +362,17 @@ def _has_organism_field(orm: type[Record]) -> bool:
 def _get_organism_record(
     field: StrField, organism: str | Record, force: bool = False
 ) -> Record:
-    model = field.field.model
+    registry = field.field.model
     check = True
-    if not force and hasattr(model, "_ontology_id_field"):
-        check = field.field.name != model._ontology_id_field
+    if not force and hasattr(registry, "_ontology_id_field"):
+        check = field.field.name != registry._ontology_id_field
         # e.g. bionty.CellMarker has "name" as _ontology_id_field
-        if not model._ontology_id_field.endswith("id"):
+        if not registry._ontology_id_field.endswith("id"):
             check = True
 
-    if _has_organism_field(model) and check:
+    if _has_organism_field(registry) and check:
         from bionty._bionty import create_or_get_organism_record
 
-        organism_record = create_or_get_organism_record(organism=organism, orm=model)
+        organism_record = create_or_get_organism_record(organism=organism, orm=registry)
         if organism_record is not None:
             return organism_record

lamindb/_query_set.py

@@ -243,10 +243,10 @@ class QuerySet(models.QuerySet, CanValidate):
         else:
             raise MultipleResultsFound(self.all())
 
-    def latest_version(self) -> RecordsList:
+    def latest_version(self) -> QuerySet:
         """Filter every version family by latest version."""
         if issubclass(self.model, IsVersioned):
-            return filter_query_set_by_latest_version(self)
+            return self.filter(is_latest=True)
         else:
             raise ValueError("Record isn't subclass of `lamindb.core.IsVersioned`")
 
@@ -288,29 +288,6 @@ class QuerySet(models.QuerySet, CanValidate):
         return _standardize(cls=self, values=values, field=field, **kwargs)
 
 
-def filter_query_set_by_latest_version(ordered_query_set: QuerySet) -> RecordsList:
-    # evaluating length can be very costly, hence, the try-except block
-    try:
-        first_record = ordered_query_set[0]
-    except IndexError:
-        return ordered_query_set
-    records_in_view = {}
-    records_in_view[first_record.stem_uid] = first_record
-    for record in ordered_query_set:
-        # this overwrites user-provided ordering (relevant records ordered by a
-        # certain field will not show if they are not the latest version)
-        if record.stem_uid not in records_in_view:
-            records_in_view[record.stem_uid] = record
-        else:
-            if record.created_at > records_in_view[record.stem_uid].created_at:
-                # deleting the entry is needed to preserve the integrity of
-                # user-provided ordering
-                del records_in_view[record.stem_uid]
-                records_in_view[record.stem_uid] = record
-    list_records_in_view = RecordsList(records_in_view.values())
-    return list_records_in_view
-
-
 models.QuerySet.df = QuerySet.df
 models.QuerySet.list = QuerySet.list
 models.QuerySet.first = QuerySet.first

lamindb/_record.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, NamedTuple
 
 import dj_database_url
 import lamindb_setup as ln_setup
-from django.db import connections
+from django.db import connections, transaction
 from django.db.models import IntegerField, Manager, Q, QuerySet, Value
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
@@ -36,9 +36,9 @@ def init_self_from_db(self: Record, existing_record: Record):
     self._state.db = "default"
 
 
-def validate_required_fields(orm: Record, kwargs):
+def validate_required_fields(record: Record, kwargs):
     required_fields = {
-        k.name for k in orm._meta.fields if not k.null and k.default is None
+        k.name for k in record._meta.fields if not k.null and k.default is None
     }
     required_fields_not_passed = {k: None for k in required_fields if k not in kwargs}
     kwargs.update(required_fields_not_passed)
@@ -77,9 +77,9 @@ def suggest_records_with_similar_names(record: Record, kwargs) -> bool:
     return False
 
 
-def __init__(orm: Record, *args, **kwargs):
+def __init__(record: Record, *args, **kwargs):
     if not args:
-        validate_required_fields(orm, kwargs)
+        validate_required_fields(record, kwargs)
 
         # do not search for names if an id is passed; this is important
         # e.g. when synching ids from the notebook store to lamindb
@@ -87,29 +87,29 @@ def __init__(orm: Record, *args, **kwargs):
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
         if settings.creation.search_names and not has_consciously_provided_uid:
-            match = suggest_records_with_similar_names(orm, kwargs)
+            match = suggest_records_with_similar_names(record, kwargs)
             if match:
                 if "version" in kwargs:
                     version_comment = " and version"
-                    existing_record = orm.__class__.filter(
+                    existing_record = record.__class__.filter(
                         name=kwargs["name"], version=kwargs["version"]
                     ).one_or_none()
                 else:
                     version_comment = ""
-                    existing_record = orm.__class__.filter(name=kwargs["name"]).one()
+                    existing_record = record.__class__.filter(name=kwargs["name"]).one()
                 if existing_record is not None:
                     logger.important(
-                        f"returning existing {orm.__class__.__name__} record with same"
+                        f"returning existing {record.__class__.__name__} record with same"
                         f" name{version_comment}: '{kwargs['name']}'"
                     )
-                    init_self_from_db(orm, existing_record)
+                    init_self_from_db(record, existing_record)
                     return None
-        super(Record, orm).__init__(**kwargs)
-    elif len(args) != len(orm._meta.concrete_fields):
+        super(Record, record).__init__(**kwargs)
+    elif len(args) != len(record._meta.concrete_fields):
         raise ValueError("please provide keyword arguments, not plain arguments")
     else:
         # object is loaded from DB (**kwargs could be omitted below, I believe)
-        super(Record, orm).__init__(*args, **kwargs)
+        super(Record, record).__init__(*args, **kwargs)
 
 
 @classmethod  # type:ignore
@@ -132,7 +132,10 @@ def get(cls, idlike: int | str) -> Record:
     else:
         qs = filter(cls, uid__startswith=idlike)
         if issubclass(cls, IsVersioned):
-            return qs.latest_version().one()
+            if len(idlike) <= cls._len_stem_uid:
+                return qs.latest_version().one()
+            else:
+                return qs.one()
         else:
             return qs.one()
 
@@ -165,9 +168,7 @@ def from_values(
 ) -> list[Record]:
     """{}"""  # noqa: D415
     from_source = True if cls.__module__.startswith("bionty.") else False
-    # if records from source is already saved in db, skip from_source
-    if isinstance(source, Record) and source.in_db:
-        from_source = False
+
     field_str = get_name_field(cls, field=field)
     return get_or_create_records(
         iterable=values,
@@ -191,11 +192,11 @@ def _search(
     truncate_words: bool = False,
 ) -> QuerySet:
     input_queryset = _queryset(cls, using_key=using_key)
-    orm = input_queryset.model
+    registry = input_queryset.model
     if field is None:
         fields = [
             field.name
-            for field in orm._meta.fields
+            for field in registry._meta.fields
             if field.get_internal_type() in {"CharField", "TextField"}
         ]
     else:
@@ -287,7 +288,7 @@ def _lookup(
 ) -> NamedTuple:
     """{}"""  # noqa: D415
     queryset = _queryset(cls, using_key=using_key)
-    field = get_name_field(orm=queryset.model, field=field)
+    field = get_name_field(registry=queryset.model, field=field)
 
     return Lookup(
         records=queryset,
@@ -296,7 +297,7 @@ def _lookup(
         prefix="ln",
     ).lookup(
         return_field=(
-            get_name_field(orm=queryset.model, field=return_field)
+            get_name_field(registry=queryset.model, field=return_field)
             if return_field is not None
             else None
         )
@@ -315,24 +316,24 @@ def lookup(
 
 
 def get_name_field(
-    orm: Record | QuerySet | Manager,
+    registry: type[Record] | QuerySet | Manager,
     *,
     field: str | StrField | None = None,
 ) -> str:
-    """Get the 1st char or text field from the orm."""
-    if isinstance(orm, (QuerySet, Manager)):
-        orm = orm.model
-    model_field_names = [i.name for i in orm._meta.fields]
+    """Get the 1st char or text field from the registry."""
+    if isinstance(registry, (QuerySet, Manager)):
+        registry = registry.model
+    model_field_names = [i.name for i in registry._meta.fields]
 
     # set to default name field
     if field is None:
-        if hasattr(orm, "_name_field"):
-            field = orm._meta.get_field(orm._name_field)
+        if hasattr(registry, "_name_field"):
+            field = registry._meta.get_field(registry._name_field)
         elif "name" in model_field_names:
-            field = orm._meta.get_field("name")
+            field = registry._meta.get_field("name")
         else:
             # first char or text field that doesn't contain "id"
-            for i in orm._meta.fields:
+            for i in registry._meta.fields:
                 if "id" in i.name:
                     continue
                 if i.get_internal_type() in {"CharField", "TextField"}:
@@ -360,7 +361,7 @@ def get_name_field(
 def _queryset(cls: Record | QuerySet | Manager, using_key: str) -> QuerySet:
     if isinstance(cls, (QuerySet, Manager)):
         return cls.all()
-    elif using_key is None:
+    elif using_key is None or using_key == "default":
         return cls.objects.all()
     else:
         # using must be called on cls, otherwise the connection isn't found
@@ -528,7 +529,28 @@ def save(self, *args, **kwargs) -> Record:
     if result is not None:
         init_self_from_db(self, result)
     else:
-        super(Record, self).save(*args, **kwargs)
+        # save versioned record
+        if isinstance(self, IsVersioned) and self._is_new_version_of is not None:
+            if self._is_new_version_of.is_latest:
+                is_new_version_of = self._is_new_version_of
+            else:
+                # need one additional request
+                is_new_version_of = self.__class__.objects.get(
+                    is_latest=True, uid__startswith=self.stem_uid
+                )
+                logger.warning(
+                    f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
+                )
+            is_new_version_of.is_latest = False
+            with transaction.atomic():
+                is_new_version_of._is_new_version_of = (
+                    None  # ensure we don't start a recursion
+                )
+                is_new_version_of.save()
+                super(Record, self).save(*args, **kwargs)
+        # save unversioned record
+        else:
+            super(Record, self).save(*args, **kwargs)
     # perform transfer of many-to-many fields
     # only supported for Artifact and Collection records
     if db is not None and db != "default" and using_key is None:
@@ -553,6 +575,30 @@ def save(self, *args, **kwargs) -> Record:
     return self
 
 
+def delete(self) -> None:
+    """Delete the record."""
+    # note that the logic below does not fire if a record is moved to the trash
+    # the idea is that moving a record to the trash should move its entire version family
+    # to the trash, whereas permanently deleting should default to only deleting a single record
+    # of a version family
+    # we can consider making it easy to permanently delete entire version families as well,
+    # but that's for another time
+    if isinstance(self, IsVersioned) and self.is_latest:
+        new_latest = (
+            self.__class__.filter(is_latest=False, uid__startswith=self.stem_uid)
+            .order_by("-created_at")
+            .first()
+        )
+        if new_latest is not None:
+            new_latest.is_latest = True
+            with transaction.atomic():
+                new_latest.save()
+                super(Record, self).delete()
+            logger.warning(f"new latest version is {new_latest}")
+            return None
+    super(Record, self).delete()
+
+
 METHOD_NAMES = [
     "__init__",
     "filter",
@@ -561,6 +607,7 @@ METHOD_NAMES = [
     "search",
     "lookup",
     "save",
+    "delete",
     "from_values",
     "using",
 ]

lamindb/_run.py

@@ -42,7 +42,7 @@ def delete_run_artifacts(run: Run) -> None:
         run.save()
     if environment is not None:
         # only delete if there are no other runs attached to this environment
-        if environment.environment_of.count() == 0:
+        if environment._environment_of.count() == 0:
             environment.delete(permanent=True)
     if report is not None:
         report.delete(permanent=True)

lamindb/_save.py

@@ -108,21 +108,21 @@ def bulk_create(records: Iterable[Record], ignore_conflicts: bool | None = False
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
-    for orm, records in records_by_orm.items():
-        orm.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
+    for registry, records in records_by_orm.items():
+        registry.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
 
 
 def bulk_update(records: Iterable[Record], ignore_conflicts: bool | None = False):
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
-    for orm, records in records_by_orm.items():
+    for registry, records in records_by_orm.items():
         field_names = [
             field.name
-            for field in orm._meta.fields
+            for field in registry._meta.fields
             if (field.name != "created_at" and field.name != "id")
         ]
-        orm.objects.bulk_update(records, field_names)
+        registry.objects.bulk_update(records, field_names)
 
 
 # This is also used within Artifact.save()

lamindb/_transform.py

@@ -22,7 +22,6 @@ def __init__(transform: Transform, *args, **kwargs):
     is_new_version_of: Transform | None = (
         kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
     )
-    (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
@@ -55,6 +54,7 @@ def __init__(transform: Transform, *args, **kwargs):
         reference=reference,
         reference_type=reference_type,
         _has_consciously_provided_uid=has_consciously_provided_uid,
+        is_new_version_of=is_new_version_of,
     )
 
 

lamindb/_view.py

@@ -41,15 +41,17 @@ def view(
         schema_module = importlib.import_module(get_schema_module_name(schema_name))
 
         all_registries = {
-            orm
-            for orm in schema_module.__dict__.values()
-            if inspect.isclass(orm)
-            and issubclass(orm, Record)
-            and orm.__name__ != "Record"
+            registry
+            for registry in schema_module.__dict__.values()
+            if inspect.isclass(registry)
+            and issubclass(registry, Record)
+            and registry is not Record
         }
         if registries is not None:
             filtered_registries = {
-                orm for orm in all_registries if orm.__name__ in registries
+                registry
+                for registry in all_registries
+                if registry.__name__ in registries
             }
         else:
             filtered_registries = all_registries
@@ -59,12 +61,12 @@ def view(
             logger.print("*" * len(section_no_color))
             logger.print(section)
             logger.print("*" * len(section_no_color))
-        for orm in sorted(filtered_registries, key=lambda x: x.__name__):
-            if hasattr(orm, "updated_at"):
-                df = orm.filter().order_by("-updated_at")[:n].df()
+        for registry in sorted(filtered_registries, key=lambda x: x.__name__):
+            if hasattr(registry, "updated_at"):
+                df = registry.filter().order_by("-updated_at")[:n].df()
             else:
                 # need to adjust in the future
-                df = orm.df().iloc[-n:]
+                df = registry.df().iloc[-n:]
             if df.shape[0] > 0:
-                logger.print(colors.blue(colors.bold(orm.__name__)))
+                logger.print(colors.blue(colors.bold(registry.__name__)))
                 show(df)

lamindb/core/__init__.py

@@ -6,6 +6,7 @@ Registries:
    :toctree: .
 
    Record
+   Registry
    QuerySet
    QueryManager
    RecordsList
@@ -66,6 +67,7 @@ from lnschema_core.models import (
     IsVersioned,
     ParamValue,
     Record,
+    Registry,
     TracksRun,
     TracksUpdates,
 )

lamindb/core/_data.py

@@ -14,8 +14,8 @@ from lnschema_core.models import (
     Record,
     Run,
     ULabel,
-    __repr__,
     format_field_value,
+    record_repr,
 )
 
 from lamindb._parents import view_lineage
@@ -108,7 +108,7 @@ def describe(self: HasFeatures, print_types: bool = False):
     # )
 
     model_name = self.__class__.__name__
-    msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}\n"
+    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
     prov_msg = ""
 
     fields = self._meta.fields
@@ -251,8 +251,8 @@ def add_labels(
         if feature.dtype.startswith("cat["):
             orm_dict = dict_schema_name_to_model_name(Artifact)
             for reg in feature.dtype.replace("cat[", "").rstrip("]").split("|"):
-                orm = orm_dict.get(reg)
-                records_validated += orm.from_values(records, field=field)
+                registry = orm_dict.get(reg)
+                records_validated += registry.from_values(records, field=field)
 
         # feature doesn't have registries and therefore can't create records from values
         # ask users to pass records

lamindb/core/_feature_manager.py

@@ -118,9 +118,7 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
 
 def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
     link_model_name = link.__class__.__name__
-    if (
-        link_model_name == "ModelBase" or link_model_name == "RecordMeta"
-    ):  # we passed the type of the link
+    if link_model_name in {"Registry", "ModelBase"}:  # we passed the type of the link
         link_model_name = link.__name__
     return link_model_name.replace(data.__class__.__name__, "").lower()
 
@@ -746,9 +744,9 @@ def _add_set_from_mudata(
     # parse and register features
     mdata = self._host.load()
     feature_sets = {}
-    obs_features = features = Feature.from_values(mdata.obs.columns)
+    obs_features = Feature.from_values(mdata.obs.columns)
     if len(obs_features) > 0:
-        feature_sets["obs"] = FeatureSet(features=features)
+        feature_sets["obs"] = FeatureSet(features=obs_features)
     for modality, field in var_fields.items():
         modality_fs = parse_feature_sets_from_anndata(
             mdata[modality],
@@ -760,8 +758,20 @@ def _add_set_from_mudata(
         for k, v in modality_fs.items():
             feature_sets[f"['{modality}'].{k}"] = v
 
+    def unify_feature_sets_by_hash(feature_sets):
+        unique_values = {}
+
+        for key, value in feature_sets.items():
+            value_hash = value.hash  # Assuming each value has a .hash attribute
+            if value_hash in unique_values:
+                feature_sets[key] = unique_values[value_hash]
+            else:
+                unique_values[value_hash] = value
+
+        return feature_sets
+
     # link feature sets
-    self._host._feature_sets = feature_sets
+    self._host._feature_sets = unify_feature_sets_by_hash(feature_sets)
     self._host.save()
 
 

lamindb/core/_label_manager.py

@@ -32,14 +32,15 @@ def get_labels_as_dict(self: HasFeatures, links: bool = False):
         "input_of_runs",
         "collections",
         "_source_code_artifact_of",
-        "report_of",
-        "environment_of",
+        "_report_of",
+        "_environment_of",
         "links_collection",
         "links_artifact",
         "links_feature_set",
         "previous_runs",
         "_feature_values",
-        "_lnschema_core_collection__actions_+",
+        "_action_targets",
+        "_lnschema_core_collection__actions_+",  # something seems off with this one
         "_actions",
     }
     labels = {}  # type: ignore