lamindb 0.74.2__py3-none-any.whl → 0.75.0__py3-none-any.whl

lamindb/core/_feature_manager.py

@@ -39,7 +39,7 @@ from lamindb._feature import FEATURE_TYPES, convert_numpy_dtype_to_lamin_feature
 from lamindb._feature_set import DICT_KEYS_TYPE, FeatureSet
 from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
-    get_default_str_field,
+    get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
 )
@@ -88,12 +88,12 @@ def get_feature_set_by_slot_(host) -> dict:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     # otherwise, we need a query
-    feature_set_links = (
+    links_feature_set = (
         host.feature_sets.through.objects.using(host_db)
         .filter(**kwargs)
         .select_related("featureset")
     )
-    return {fsl.slot: fsl.featureset for fsl in feature_set_links}
+    return {fsl.slot: fsl.featureset for fsl in links_feature_set}
 
 
 def get_label_links(
@@ -112,8 +112,8 @@ def get_label_links(
 def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
-    feature_set_links = host.feature_sets.through.objects.filter(**kwargs)
-    return feature_set_links
+    links_feature_set = host.feature_sets.through.objects.filter(**kwargs)
+    return links_feature_set
 
 
 def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
@@ -122,12 +122,7 @@ def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
         link_model_name == "ModelBase" or link_model_name == "RecordMeta"
     ):  # we passed the type of the link
         link_model_name = link.__name__
-    link_attr = link_model_name.replace(data.__class__.__name__, "")
-    if link_attr == "ExperimentalFactor":
-        link_attr = "experimental_factor"
-    else:
-        link_attr = link_attr.lower()
-    return link_attr
+    return link_model_name.replace(data.__class__.__name__, "").lower()
 
 
 # Custom aggregation for SQLite
@@ -182,14 +177,14 @@ def print_features(
     non_labels_msg = ""
     if self.id is not None and self.__class__ == Artifact or self.__class__ == Run:
         attr_name = "param" if print_params else "feature"
-        feature_values = (
-            getattr(self, f"{attr_name}_values")
+        _feature_values = (
+            getattr(self, f"_{attr_name}_values")
             .values(f"{attr_name}__name", f"{attr_name}__dtype")
             .annotate(values=custom_aggregate("value", self._state.db))
             .order_by(f"{attr_name}__name")
         )
-        if len(feature_values) > 0:
-            for fv in feature_values:
+        if len(_feature_values) > 0:
+            for fv in _feature_values:
                 feature_name = fv[f"{attr_name}__name"]
                 feature_dtype = fv[f"{attr_name}__dtype"]
                 values = fv["values"]
@@ -217,7 +212,7 @@ def print_features(
         for slot, feature_set in get_feature_set_by_slot_(self).items():
             features = feature_set.members
             # features.first() is a lot slower than features[0] here
-            name_field = get_default_str_field(features[0])
+            name_field = get_name_field(features[0])
             feature_names = list(features.values_list(name_field, flat=True)[:20])
             type_str = f": {feature_set.registry}" if print_types else ""
             feature_set_msg += (
@@ -246,7 +241,7 @@ def parse_feature_sets_from_anndata(
             from lamindb.core.storage._backed_access import backed_access
 
             using_key = settings._using_key
-            data_parse = backed_access(filepath, using_key)
+            data_parse = backed_access(filepath, using_key=using_key)
         else:
             data_parse = ad.read_h5ad(filepath, backed="r")
         type = "float"
@@ -316,13 +311,13 @@ def infer_feature_type_convert_json(
         if len(value) > 0:  # type: ignore
             first_element_type = type(next(iter(value)))
             if all(isinstance(elem, first_element_type) for elem in value):
-                if first_element_type == bool:
+                if first_element_type is bool:
                     return f"list[{FEATURE_TYPES['bool']}]", value
-                elif first_element_type == int:
+                elif first_element_type is int:
                     return f"list[{FEATURE_TYPES['int']}]", value
-                elif first_element_type == float:
+                elif first_element_type is float:
                     return f"list[{FEATURE_TYPES['float']}]", value
-                elif first_element_type == str:
+                elif first_element_type is str:
                     if str_as_ulabel:
                         return FEATURE_TYPES["str"] + "[ULabel]", value
                     else:
@@ -390,7 +385,7 @@ def filter(cls, **expression) -> QuerySet:
         feature = features.get(name=normalized_key)
         if not feature.dtype.startswith("cat"):
             feature_value = value_model.filter(feature=feature, value=value).one()
-            new_expression["feature_values"] = feature_value
+            new_expression["_feature_values"] = feature_value
         else:
             if isinstance(value, str):
                 label = ULabel.filter(name=value).one()
@@ -478,7 +473,7 @@ def _add_values(
     )
     # figure out which of the values go where
     features_labels = defaultdict(list)
-    feature_values = []
+    _feature_values = []
     not_validated_values = []
     for key, value in features_values.items():
         feature = model.filter(name=key).one()
@@ -508,7 +503,7 @@ def _add_values(
             feature_value = value_model.filter(**filter_kwargs).one_or_none()
             if feature_value is None:
                 feature_value = value_model(**filter_kwargs)
-            feature_values.append(feature_value)
+            _feature_values.append(feature_value)
         else:
             if isinstance(value, Record) or (
                 isinstance(value, Iterable) and isinstance(next(iter(value)), Record)
@@ -578,7 +573,7 @@ def _add_values(
             except Exception:
                 save(links, ignore_conflicts=True)
                 # now deal with links that were previously saved without a feature_id
-                saved_links = LinkORM.filter(
+                links_saved = LinkORM.filter(
                     **{
                         "artifact_id": self._host.id,
                         f"{field_name}__in": [
@@ -586,7 +581,7 @@ def _add_values(
                         ],
                     }
                 )
-                for link in saved_links.all():
+                for link in links_saved.all():
                     # TODO: also check for inconsistent features
                     if link.feature_id is None:
                         link.feature_id = [
@@ -595,13 +590,13 @@ def _add_values(
                             if l.id == getattr(link, field_name)
                         ][0]
                         link.save()
-    if feature_values:
-        save(feature_values)
+    if _feature_values:
+        save(_feature_values)
         if is_param:
-            LinkORM = self._host.param_values.through
+            LinkORM = self._host._param_values.through
             valuefield_id = "paramvalue_id"
         else:
-            LinkORM = self._host.feature_values.through
+            LinkORM = self._host._feature_values.through
             valuefield_id = "featurevalue_id"
         links = [
             LinkORM(
@@ -610,7 +605,7 @@ def _add_values(
                     valuefield_id: feature_value.id,
                 }
             )
-            for feature_value in feature_values
+            for feature_value in _feature_values
         ]
         # a link might already exist, to avoid raising a unique constraint
         # error, ignore_conflicts
@@ -683,10 +678,10 @@ def _add_set_from_df(
 ):
     """Add feature set corresponding to column names of DataFrame."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "DataFrame"  # noqa: S101
+        assert self._host._accessor == "DataFrame"  # noqa: S101
     else:
         # Collection
-        assert self._host.artifact.accessor == "DataFrame"  # noqa: S101
+        assert self._host.artifact._accessor == "DataFrame"  # noqa: S101
 
     # parse and register features
     registry = field.field.model
@@ -714,7 +709,7 @@ def _add_set_from_anndata(
 ):
     """Add features from AnnData."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "AnnData"  # noqa: S101
+        assert self._host._accessor == "AnnData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -744,7 +739,7 @@ def _add_set_from_mudata(
     if obs_fields is None:
         obs_fields = {}
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "MuData"  # noqa: S101
+        assert self._host._accessor == "MuData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -770,7 +765,7 @@ def _add_set_from_mudata(
     self._host.save()
 
 
-def _add_from(self, data: HasFeatures, parents: bool = True):
+def _add_from(self, data: HasFeatures):
     """Transfer features from a artifact or collection."""
     # This only covers feature sets, though.
     using_key = settings._using_key
@@ -781,24 +776,14 @@ def _add_from(self, data: HasFeatures, parents: bool = True):
         registry = members[0].__class__
         # note here the features are transferred based on an unique field
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-        # TODO: get a default ID field for the registry
-        if hasattr(registry, "ontology_id") and parents:
-            field = "ontology_id"
-        elif hasattr(registry, "ensembl_gene_id"):
-            field = "ensembl_gene_id"
-        elif hasattr(registry, "uniprotkb_id"):
-            field = "uniprotkb_id"
-
-        if registry.__get_name_with_schema__() == "bionty.Organism":
-            parents = False
+        if hasattr(registry, "_ontology_id_field"):
+            field = registry._ontology_id_field
         # this will be e.g. be a list of ontology_ids or uids
         member_uids = list(members.values_list(field, flat=True))
-        # create records from ontology_id in order to populate parents
-        if field == "ontology_id" and len(member_uids) > 0 and parents:
+        # create records from ontology_id
+        if hasattr(registry, "_ontology_id_field") and len(member_uids) > 0:
             # create from bionty
-            records = registry.from_values(member_uids, field=field)
-            if len(records) > 0:
-                save(records, parents=parents)
+            save(registry.from_values(member_uids, field=field))
         validated = registry.validate(member_uids, field=field, mute=True)
         new_members_uids = list(compress(member_uids, ~validated))
         new_members = members.filter(**{f"{field}__in": new_members_uids}).all()
@@ -814,14 +799,14 @@ def _add_from(self, data: HasFeatures, parents: bool = True):
                 # in the previous step transfer_fk_to_default_db_bulk
                 transfer_to_default_db(feature, using_key, mute=mute, transfer_fk=False)
             logger.info(f"saving {n_new_members} new {registry.__name__} records")
-            save(new_members, parents=parents)
+            save(new_members)
 
         # create a new feature set from feature values using the same uid
         feature_set_self = FeatureSet.from_values(
             member_uids, field=getattr(registry, field)
         )
         if feature_set_self is None:
-            if hasattr(registry, "organism"):
+            if hasattr(registry, "organism_id"):
                 logger.warning(
                     f"FeatureSet is not transferred, check if organism is set correctly: {feature_set}"
                 )

lamindb/core/_label_manager.py

@@ -4,13 +4,13 @@ from collections import defaultdict
 from typing import TYPE_CHECKING, Dict
 
 import numpy as np
-from lamin_utils import colors, logger
+from lamin_utils import colors
 from lnschema_core.models import Feature
 
 from lamindb._from_values import _print_values
 from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
-    get_default_str_field,
+    get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
 )
@@ -28,17 +28,19 @@ if TYPE_CHECKING:
 def get_labels_as_dict(self: HasFeatures, links: bool = False):
     exclude_set = {
         "feature_sets",
-        "unordered_artifacts",
-        "input_of",
+        "artifacts",
+        "input_of_runs",
         "collections",
-        "source_code_of",
+        "_source_code_artifact_of",
         "report_of",
         "environment_of",
-        "collection_links",
-        "artifact_links",
-        "feature_set_links",
+        "links_collection",
+        "links_artifact",
+        "links_feature_set",
         "previous_runs",
-        "feature_values",
+        "_feature_values",
+        "_lnschema_core_collection__actions_+",
+        "_actions",
     }
     labels = {}  # type: ignore
     if self.id is None:
@@ -62,7 +64,7 @@ def print_labels(self: HasFeatures, field: str = "name", print_types: bool = Fal
         try:
             labels_list = list(labels.values_list(field, flat=True))
             if len(labels_list) > 0:
-                get_default_str_field(labels)
+                get_name_field(labels)
                 print_values = _print_values(labels_list, n=10)
                 type_str = f": {related_model}" if print_types else ""
                 labels_msg += f"    .{related_name}{type_str} = {print_values}\n"
@@ -76,32 +78,24 @@ def print_labels(self: HasFeatures, field: str = "name", print_types: bool = Fal
 
 
 # Alex: is this a label transfer function?
-def validate_labels(labels: QuerySet | list | dict, parents: bool = True):
+def validate_labels(labels: QuerySet | list | dict):
     def validate_labels_registry(
-        labels: QuerySet | list | dict, parents: bool = True
+        labels: QuerySet | list | dict,
     ) -> tuple[list[str], list[str]]:
         if len(labels) == 0:
             return [], []
         registry = labels[0].__class__
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-        if hasattr(registry, "ontology_id") and parents:
-            field = "ontology_id"
-        elif hasattr(registry, "ensembl_gene_id"):
-            field = "ensembl_gene_id"
-        elif hasattr(registry, "uniprotkb_id"):
-            field = "uniprotkb_id"
-        if registry.__get_name_with_schema__() == "bionty.Organism":
-            parents = False
+        if hasattr(registry, "_ontology_id_field"):
+            field = registry._ontology_id_field
         # if the field value is None, use uid field
         label_uids = np.array(
             [getattr(label, field) for label in labels if label is not None]
         )
-        # save labels from ontology_ids so that parents are populated
-        if field == "ontology_id" and len(label_uids) > 0:
+        # save labels from ontology_ids
+        if hasattr(registry, "_ontology_id_field") and len(label_uids) > 0:
             try:
-                records = registry.from_values(label_uids, field=field)
-                if len(records) > 0:
-                    save(records, parents=parents)
+                save(registry.from_values(label_uids, field=field))
             except Exception:  # noqa S110
                 pass
             field = "uid"
@@ -117,11 +111,9 @@ def validate_labels(labels: QuerySet | list | dict, parents: bool = True):
     if isinstance(labels, Dict):
         result = {}
         for registry, labels_registry in labels.items():
-            result[registry] = validate_labels_registry(
-                labels_registry, parents=parents
-            )
+            result[registry] = validate_labels_registry(labels_registry)
     else:
-        return validate_labels_registry(labels, parents=parents)
+        return validate_labels_registry(labels)
 
 
 class LabelManager:
@@ -176,7 +168,7 @@ class LabelManager:
 
         return get_labels(self._host, feature=feature, mute=mute, flat_names=flat_names)
 
-    def add_from(self, data: HasFeatures, parents: bool = True) -> None:
+    def add_from(self, data: HasFeatures) -> None:
         """Add labels from an artifact or collection to another artifact or collection.
 
         Examples:
@@ -202,15 +194,15 @@ class LabelManager:
                 data_name_lower = data.__class__.__name__.lower()
                 labels_by_features = defaultdict(list)
                 features = set()
-                _, new_labels = validate_labels(labels, parents=parents)
+                _, new_labels = validate_labels(labels)
                 if len(new_labels) > 0:
                     transfer_fk_to_default_db_bulk(new_labels, using_key)
                 for label in labels:
                     # if the link table doesn't follow this convention, we'll ignore it
-                    if not hasattr(label, f"{data_name_lower}_links"):
+                    if not hasattr(label, f"links_{data_name_lower}"):
                         key = None
                     else:
-                        link = getattr(label, f"{data_name_lower}_links").get(
+                        link = getattr(label, f"links_{data_name_lower}").get(
                             **{f"{data_name_lower}_id": data.id}
                         )
                         if link.feature is not None:
@@ -237,7 +229,7 @@ class LabelManager:
                         transfer_to_default_db(
                             feature, using_key, mute=True, transfer_fk=False
                         )
-                    save(new_features, parents=parents)
+                    save(new_features)
                 if hasattr(self._host, related_name):
                     for feature_name, labels in labels_by_features.items():
                         if feature_name is not None:

lamindb/core/_mapped_collection.py

@@ -10,7 +10,7 @@ import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
 
-from .storage._backed_access import (
+from .storage._anndata_accessor import (
     ArrayType,
     ArrayTypes,
     GroupType,

lamindb/core/_run_context.py

@@ -11,7 +11,6 @@ from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 from lnschema_core import Run, Transform, ids
 from lnschema_core.models import Param, ParamValue, RunParamValue
-from lnschema_core.types import TransformType
 from lnschema_core.users import current_user_id
 
 from ._settings import settings
@@ -27,6 +26,7 @@ from .versioning import bump_version as bump_version_function
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
+    from lnschema_core.types import TransformType
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -279,14 +279,14 @@ class run_context:
                 ).one_or_none()
                 if is_run_from_ipython:
                     key, name = cls._track_notebook(path=path)
-                    transform_type = TransformType.notebook
+                    transform_type = "notebook"
                     transform_ref = None
                     transform_ref_type = None
                 else:
                     (name, key, transform_ref, transform_ref_type) = cls._track_script(
                         path=path
                     )
-                    transform_type = TransformType.script
+                    transform_type = "script"
                 # overwrite whatever is auto-detected in the notebook or script
                 if transform_settings.name is not None:
                     name = transform_settings.name
@@ -323,9 +323,7 @@ class run_context:
             cls.transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
-            new_run = (
-                False if cls.transform.type == TransformType.notebook.value else True
-            )  # type: ignore
+            new_run = False if cls.transform.type == "notebook" else True  # type: ignore
 
         run = None
         from lamindb._run import Run
@@ -479,7 +477,7 @@ class run_context:
                     transform.save()
                     logger.important(f"updated: {transform}")
             # check whether transform source code was already saved
-            if transform.source_code_id is not None:
+            if transform._source_code_artifact_id is not None:
                 response = None
                 if is_run_from_ipython:
                     if os.getenv("LAMIN_TESTING") is None:
@@ -491,7 +489,7 @@ class run_context:
                         response = "y"
                 else:
                     hash, _ = hash_file(cls.path)  # ignore hash_type for now
-                    if hash != transform.source_code.hash:
+                    if hash != transform._source_code_artifact.hash:
                         # only if hashes don't match, we need user input
                         if os.getenv("LAMIN_TESTING") is None:
                             response = input(

lamindb/core/datasets/_core.py

@@ -83,7 +83,7 @@ def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts(
         ln.settings.verbosity = "error"
         ln.Feature(name="assay", dtype=[bt.ExperimentalFactor]).save()
         ln.Feature(name="organism", dtype=[bt.Organism]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008896").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008896").save()
         ln.settings.verbosity = verbosity
 
     return Path(filepath)
@@ -185,19 +185,17 @@ def anndata_mouse_sc_lymph_node(
 
         verbosity = ln.settings.verbosity
         ln.settings.verbosity = "error"
-        auto_save_parents = bt.settings.auto_save_parents
-        bt.settings.auto_save_parents = False
         # strain
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0004472").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0004472").save()
         # developmental stage
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0001272").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0001272").save()
         # tissue
-        bt.Tissue.from_public(ontology_id="UBERON:0001542").save()
+        bt.Tissue.from_source(ontology_id="UBERON:0001542").save()
         # cell types
         ln.save(bt.CellType.from_values(["CL:0000115", "CL:0000738"], "ontology_id"))
         # assays
         ln.Feature(name="assay", dtype=[bt.ExperimentalFactor]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008913").save()
         # genes
         validated = bt.Gene.public(organism="mouse").validate(
             adata.var.index, field="ensembl_gene_id"
@@ -215,7 +213,6 @@ def anndata_mouse_sc_lymph_node(
             labels += [ln.ULabel(name=name) for name in adata.obs[col]]
         ln.save(labels)
         ln.settings.verbosity = verbosity
-        bt.settings.auto_save_parents = auto_save_parents
 
     return adata
 
@@ -313,8 +310,6 @@ def anndata_human_immune_cells(
 
         verbosity = ln.settings.verbosity
         ln.settings.verbosity = "error"
-        auto_save_parents = bt.settings.auto_save_parents
-        bt.settings.auto_save_parents = False
         ln.save(
             bt.Gene.from_values(
                 adata.var.index, field="ensembl_gene_id", organism="human"
@@ -328,17 +323,16 @@ def anndata_human_immune_cells(
         ln.Feature(name="tissue", dtype=[bt.Tissue]).save()
         ln.Feature(name="organism", dtype=[bt.Organism]).save()
         ln.Feature(name="donor", dtype=[ln.ULabel]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008913").save()
         ln.save([ln.ULabel(name=name) for name in adata.obs.donor.unique()])
         ln.settings.verbosity = verbosity
-        bt.settings.auto_save_parents = auto_save_parents
     return adata
 
 
 def anndata_with_obs() -> ad.AnnData:
     """Create a mini anndata with cell_type, disease and tissue."""
     import anndata as ad
-    import bionty_base
+    import bionty.base as bionty_base
 
     celltypes = ["T cell", "hematopoietic stem cell", "hepatocyte", "my new cell type"]
     celltype_ids = ["CL:0000084", "CL:0000037", "CL:0000182", ""]

lamindb/core/exceptions.py

@@ -10,6 +10,7 @@ The registry base class:
    NoTitleError
    MissingTransformSettings
    UpdateTransformSettings
+   IntegrityError
 
 """
 
@@ -25,6 +26,16 @@ class ValidationError(SystemExit):
 # -------------------------------------------------------------------------------------
 
 
+class IntegrityError(Exception):
+    """Integrity error.
+
+    For instance, it's not allowed to delete artifacts outside managed storage
+    locations.
+    """
+
+    pass
+
+
 class NotebookNotSavedError(Exception):
     """Notebook wasn't saved."""
 

lamindb/core/storage/__init__.py

@@ -6,6 +6,7 @@
    AnnDataAccessor
    BackedAccessor
 """
+
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
 from ._anndata_sizes import size_adata