lamindb 0.64.2__py3-none-any.whl → 0.65.1__py3-none-any.whl

lamindb/_transform.py

@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional
 
 from lnschema_core.models import TRANSFORM_TYPE_DEFAULT, Transform
 
-from .dev.versioning import get_ids_from_old_version, init_uid
+from .dev.versioning import get_uid_from_old_version, init_uid
 
 if TYPE_CHECKING:
     from lnschema_core.types import TransformType
@@ -19,9 +19,7 @@ def __init__(transform: Transform, *args, **kwargs):
     is_new_version_of: Optional[Transform] = (
         kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
     )
-    initial_version_id: Optional[int] = (
-        kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None
-    )
+    (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: Optional[str] = kwargs.pop("version") if "version" in kwargs else None
     type: Optional[TransformType] = (
         kwargs.pop("type") if "type" in kwargs else TRANSFORM_TYPE_DEFAULT
@@ -37,12 +35,12 @@ def __init__(transform: Transform, *args, **kwargs):
             f" but you passed: {kwargs}"
         )
     if is_new_version_of is None:
-        new_uid = init_uid(version=version, n_full_id=14)
+        new_uid = init_uid(version=version, n_full_id=Transform._len_full_uid)
     else:
         if not isinstance(is_new_version_of, Transform):
             raise TypeError("is_new_version_of has to be of type ln.Transform")
-        new_uid, initial_version_id, version = get_ids_from_old_version(
-            is_new_version_of, version, n_full_id=14
+        new_uid, version = get_uid_from_old_version(
+            is_new_version_of, version, n_full_id=Transform._len_full_uid
         )
         if name is None:
             name = is_new_version_of.name
@@ -60,7 +58,6 @@ def __init__(transform: Transform, *args, **kwargs):
         short_name=short_name,
         type=type,
         version=version,
-        initial_version_id=initial_version_id,
         reference=reference,
         _has_consciously_provided_uid=has_consciously_provided_uid,
     )

lamindb/dev/__init__.py

@@ -24,6 +24,7 @@ Functionality of data registries:
    FeatureManager
    LabelManager
    IsTree
+   IsVersioned
 
 Functionality of metadata registries:
 
@@ -47,11 +48,18 @@ Auxiliary tools:
    Settings
    types
    exceptions
-   MappedDataset
+   MappedCollection
 """
 
 from lamin_utils._inspect import InspectResult
-from lnschema_core.models import CanValidate, Data, HasParents, IsTree, Registry
+from lnschema_core.models import (
+    CanValidate,
+    Data,
+    HasParents,
+    IsTree,
+    IsVersioned,
+    Registry,
+)
 
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet
@@ -59,6 +67,6 @@ from lamindb.dev._feature_manager import FeatureManager
 from lamindb.dev._label_manager import LabelManager
 
 from . import _data, datasets, exceptions, fields, types
-from ._mapped_dataset import MappedDataset
+from ._mapped_collection import MappedCollection
 from ._run_context import run_context
 from ._settings import Settings

lamindb/dev/_data.py

@@ -5,8 +5,8 @@ from lamin_utils import colors, logger
 from lamindb_setup.dev._docs import doc_args
 from lnschema_core.models import (
     Artifact,
+    Collection,
     Data,
-    Dataset,
     Feature,
     FeatureSet,
     Registry,
@@ -54,7 +54,7 @@ def add_transform_to_kwargs(kwargs: Dict[str, Any], run: Run):
         kwargs["transform"] = run.transform
 
 
-def save_feature_sets(self: Union[Artifact, Dataset]) -> None:
+def save_feature_sets(self: Union[Artifact, Collection]) -> None:
     if hasattr(self, "_feature_sets"):
         saved_feature_sets = {}
         for key, feature_set in self._feature_sets.items():
@@ -72,7 +72,7 @@ def save_feature_sets(self: Union[Artifact, Dataset]) -> None:
             )
 
 
-def save_feature_set_links(self: Union[Artifact, Dataset]) -> None:
+def save_feature_set_links(self: Union[Artifact, Collection]) -> None:
     from lamindb._save import bulk_create
 
     Data = self.__class__
@@ -113,10 +113,9 @@ def describe(self: Data):
         "created_by": "👤",
         "transform": _transform_emoji(self.transform),
         "run": "👣",
-        "initial_version": "🔖",
-        "file": "📄",
+        "artifact": "📄",
     }
-    if len(foreign_key_fields) > 0:  # always True for Artifact and Dataset
+    if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
         record_msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}"
         msg += f"{record_msg}\n\n"
 
@@ -209,7 +208,7 @@ def add_labels(
 ) -> None:
     """{}."""
     if self._state.adding:
-        raise ValueError("Please save the file/dataset before adding a label!")
+        raise ValueError("Please save the artifact/collection before adding a label!")
 
     if isinstance(records, (QuerySet, QuerySet.__base__)):  # need to have both
         records = records.list()
@@ -331,7 +330,7 @@ def add_labels(
                             id=old_feature_set_link.feature_set_id
                         ).one()
                         logger.info(
-                            "no file links to it anymore, deleting feature set"
+                            "nothing links to it anymore, deleting feature set"
                             f" {old_feature_set}"
                         )
                         old_feature_set.delete()
@@ -368,7 +367,7 @@ def _track_run_input(
         if run is None:
             if settings.track_run_inputs:
                 logger.hint(
-                    "you can auto-track this file as a run input by calling"
+                    "you can auto-track these data as a run input by calling"
                     " `ln.track()`"
                 )
         # assume we have a run record
@@ -390,7 +389,7 @@ def _track_run_input(
                     track_run_input = True
                 else:
                     logger.hint(
-                        "track this file as a run input by passing `is_run_input=True`"
+                        "track these data as a run input by passing `is_run_input=True`"
                     )
     else:
         track_run_input = is_run_input
@@ -409,9 +408,10 @@ def _track_run_input(
                 for data_id in input_data_ids
             ]
         else:
-            LinkORM = run.input_datasets.through
+            LinkORM = run.input_collections.through
             links = [
-                LinkORM(run_id=run.id, dataset_id=data_id) for data_id in input_data_ids
+                LinkORM(run_id=run.id, collection_id=data_id)
+                for data_id in input_data_ids
             ]
         LinkORM.objects.bulk_create(links, ignore_conflicts=True)
         # generalize below for more than one data batch

lamindb/dev/_feature_manager.py

@@ -1,8 +1,8 @@
+from itertools import compress
 from typing import Dict, Union
 
-import numpy as np
-from lamin_utils import colors
-from lnschema_core.models import Artifact, Data, Dataset, Feature
+from lamin_utils import colors, logger
+from lnschema_core.models import Artifact, Collection, Data, Feature
 
 from lamindb._feature_set import FeatureSet
 from lamindb._query_set import QuerySet
@@ -15,15 +15,15 @@ from lamindb._registry import (
 from lamindb._save import save
 
 
-def get_host_id_field(host: Union[Artifact, Dataset]) -> str:
+def get_host_id_field(host: Union[Artifact, Collection]) -> str:
     if isinstance(host, Artifact):
         host_id_field = "artifact_id"
     else:
-        host_id_field = "dataset_id"
+        host_id_field = "collection_id"
     return host_id_field
 
 
-def get_accessor_by_orm(host: Union[Artifact, Dataset]) -> Dict:
+def get_accessor_by_orm(host: Union[Artifact, Collection]) -> Dict:
     dictionary = {
         field.related_model.__get_name_with_schema__(): field.name
         for field in host._meta.related_objects
@@ -56,7 +56,7 @@ def get_feature_set_by_slot(host) -> Dict:
 
 
 def get_label_links(
-    host: Union[Artifact, Dataset], registry: str, feature: Feature
+    host: Union[Artifact, Collection], registry: str, feature: Feature
 ) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id, "feature_id": feature.id}
@@ -68,7 +68,7 @@ def get_label_links(
     return link_records
 
 
-def get_feature_set_links(host: Union[Artifact, Dataset]) -> QuerySet:
+def get_feature_set_links(host: Union[Artifact, Collection]) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     feature_set_links = host.feature_sets.through.objects.filter(**kwargs)
@@ -124,7 +124,7 @@ class FeatureManager:
     See :class:`~lamindb.dev.Data` for more information.
     """
 
-    def __init__(self, host: Union[Artifact, Dataset]):
+    def __init__(self, host: Union[Artifact, Collection]):
         self._host = host
         self._feature_set_by_slot = get_feature_set_by_slot(host)
         self._accessor_by_orm = get_accessor_by_orm(host)
@@ -160,7 +160,7 @@ class FeatureManager:
         """
         if self._host._state.adding:
             raise ValueError(
-                "Please save the artifact or dataset before adding a feature set!"
+                "Please save the artifact or collection before adding a feature set!"
             )
         host_db = self._host._state.db
         feature_set.save(using=host_db)
@@ -179,31 +179,53 @@ class FeatureManager:
             self._host.feature_sets.through(**kwargs).save(using=host_db)
             self._feature_set_by_slot[slot] = feature_set
 
-    def _add_from(self, data: Data):
-        """Transfer features from a artifact or dataset."""
+    def _add_from(self, data: Data, parents: bool = True):
+        """Transfer features from a artifact or collection."""
         for slot, feature_set in data.features._feature_set_by_slot.items():
             members = feature_set.members
+            if members.count() == 0:
+                continue
             registry = members[0].__class__
             # note here the features are transferred based on an unique field
             field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-            member_uids = np.array([getattr(member, field) for member in members])
-            validated = registry.objects.using(self._host._state.db).validate(
-                member_uids, field=field, mute=True
-            )
-            new_features = [members[int(i)] for i in np.argwhere(~validated).flatten()]
-            if len(new_features) > 0:
-                mute = True if len(new_features) > 10 else False
+            if hasattr(registry, "ontology_id") and parents:
+                field = "ontology_id"
+            if registry.__get_name_with_schema__() == "bionty.Organism":
+                parents = False
+            # this will be e.g. be a list of ontology_ids or uids
+            member_uids = list(members.values_list(field, flat=True))
+            # create records from ontology_id in order to populate parents
+            if field == "ontology_id" and len(member_uids) > 0:
+                # create from bionty
+                records = registry.from_values(member_uids, field=field)
+                if len(records) > 0:
+                    save(records, parents=parents)
+            validated = registry.validate(member_uids, field=field, mute=True)
+            new_members_uids = list(compress(member_uids, ~validated))
+            new_members = members.filter(**{f"{field}__in": new_members_uids}).all()
+            if new_members.count() > 0:
+                mute = True if new_members.count() > 10 else False
                 # transfer foreign keys needs to be run before transfer to default db
-                transfer_fk_to_default_db_bulk(new_features)
-                for feature in new_features:
+                transfer_fk_to_default_db_bulk(new_members)
+                for feature in new_members:
                     # not calling save=True here as in labels, because want to
                     # bulk save below
                     transfer_to_default_db(feature, mute=mute)
-                save(new_features)
+                logger.info(
+                    f"saving {new_members.count()} new {registry.__name__} records"
+                )
+                save(new_members, parents=parents)
 
             # create a new feature set from feature values using the same uid
             feature_set_self = FeatureSet.from_values(
                 member_uids, field=getattr(registry, field)
             )
+            if feature_set_self is None:
+                if hasattr(registry, "organism"):
+                    logger.warning(
+                        f"FeatureSet is not transferred, check if organism is set correctly: {feature_set}"
+                    )
+                continue
             feature_set_self.uid = feature_set.uid
+            logger.info(f"saving {slot} featureset: {feature_set_self}")
             self._host.features.add_feature_set(feature_set_self, slot)

lamindb/dev/_label_manager.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
 from lamin_utils import colors, logger
-from lnschema_core.models import Artifact, Data, Dataset, Feature, Registry
+from lnschema_core.models import Artifact, Collection, Data, Feature, Registry
 
 from lamindb._feature_set import dict_related_model_to_related_name
 from lamindb._from_values import _print_values
@@ -23,11 +23,12 @@ def get_labels_as_dict(self: Data):
     ).items():
         if related_name in {
             "feature_sets",
-            "files",
+            "artifacts",
             "input_of",
-            "datasets",
+            "collections",
             "source_of",
             "report_of",
+            "environment_of",
         }:
             continue
         if self.id is not None:
@@ -49,13 +50,16 @@ def print_labels(self: Data):
         return ""
 
 
-def transfer_add_labels(labels, features_lookup_self, self, row):
+def transfer_add_labels(labels, features_lookup_self, self, row, parents: bool = True):
     def transfer_single_registry(validated_labels, new_labels):
         # here the new labels are transferred to the self db
         if len(new_labels) > 0:
             transfer_fk_to_default_db_bulk(new_labels)
             for label in new_labels:
                 transfer_to_default_db(label, mute=True)
+            # not saving parents for Organism during transfer
+            registry = new_labels[0].__class__
+            logger.info(f"saving {len(new_labels)} new {registry.__name__} records")
             save(new_labels)
         # link labels records from self db
         self._host.labels.add(
@@ -64,7 +68,7 @@ def transfer_add_labels(labels, features_lookup_self, self, row):
         )
 
     # validate labels on the default db
-    result = validate_labels(labels)
+    result = validate_labels(labels, parents=parents)
     if isinstance(result, Dict):
         for _, (validated_labels, new_labels) in result.items():
             transfer_single_registry(validated_labels, new_labels)
@@ -72,17 +76,34 @@ def transfer_add_labels(labels, features_lookup_self, self, row):
         transfer_single_registry(*result)
 
 
-def validate_labels(labels: Union[QuerySet, List, Dict]):
+def validate_labels(labels: Union[QuerySet, List, Dict], parents: bool = True):
     def validate_labels_registry(
-        labels: Union[QuerySet, List, Dict],
+        labels: Union[QuerySet, List, Dict], parents: bool = True
     ) -> Tuple[List[str], List[str]]:
         if len(labels) == 0:
             return [], []
         registry = labels[0].__class__
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
+        if hasattr(registry, "ontology_id") and parents:
+            field = "ontology_id"
+        if registry.__get_name_with_schema__() == "bionty.Organism":
+            parents = False
+        # if the field value is None, use uid field
         label_uids = np.array(
             [getattr(label, field) for label in labels if label is not None]
         )
+        # save labels from ontology_ids so that parents are populated
+        if field == "ontology_id" and len(label_uids) > 0:
+            try:
+                records = registry.from_values(label_uids, field=field)
+                if len(records) > 0:
+                    save(records, parents=parents)
+            except Exception:
+                pass
+            field = "uid"
+            label_uids = np.array(
+                [getattr(label, field) for label in labels if label is not None]
+            )
         validated = registry.validate(label_uids, field=field, mute=True)
         validated_uids = label_uids[validated]
         validated_labels = registry.filter(**{f"{field}__in": validated_uids}).list()
@@ -92,9 +113,11 @@ def validate_labels(labels: Union[QuerySet, List, Dict]):
     if isinstance(labels, Dict):
         result = {}
         for registry, labels_registry in labels.items():
-            result[registry] = validate_labels_registry(labels_registry)
+            result[registry] = validate_labels_registry(
+                labels_registry, parents=parents
+            )
     else:
-        return validate_labels_registry(labels)
+        return validate_labels_registry(labels, parents=parents)
 
 
 class LabelManager:
@@ -107,7 +130,7 @@ class LabelManager:
     See :class:`~lamindb.dev.Data` for more information.
     """
 
-    def __init__(self, host: Union[Artifact, Dataset]):
+    def __init__(self, host: Union[Artifact, Collection]):
         self._host = host
 
     def __repr__(self) -> str:
@@ -150,8 +173,8 @@ class LabelManager:
 
         return get_labels(self._host, feature=feature, mute=mute, flat_names=flat_names)
 
-    def add_from(self, data: Data):
-        """Transfer labels from a file or dataset.
+    def add_from(self, data: Data, parents: bool = True):
+        """Transfer labels from a file or collection.
 
         Examples:
             >>> file1 = ln.Artifact(pd.DataFrame(index=[0, 1]))
@@ -185,13 +208,15 @@ class LabelManager:
             labels = labels.all()
             if len(labels) == 0:
                 continue
-            validated_labels, new_labels = validate_labels(labels.all())
+            validated_labels, new_labels = validate_labels(
+                labels.all(), parents=parents
+            )
             if len(new_labels) > 0:
                 transfer_fk_to_default_db_bulk(new_labels)
                 for label in new_labels:
                     transfer_to_default_db(label, mute=True)
-                save(new_labels)
-            # this should not occur as file and dataset should have the same attributes
+                save(new_labels, parents=parents)
+            # this should not occur as file and collection should have the same attributes
             # but this might not be true for custom schema
             labels_list = validated_labels + new_labels
             if hasattr(self._host, related_name):