lamindb 0.74.3__py3-none-any.whl → 0.75.1__py3-none-any.whl

lamindb/core/_feature_manager.py

@@ -39,7 +39,7 @@ from lamindb._feature import FEATURE_TYPES, convert_numpy_dtype_to_lamin_feature
 from lamindb._feature_set import DICT_KEYS_TYPE, FeatureSet
 from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
-    get_default_str_field,
+    get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
 )
@@ -88,12 +88,12 @@ def get_feature_set_by_slot_(host) -> dict:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     # otherwise, we need a query
-    feature_set_links = (
+    links_feature_set = (
         host.feature_sets.through.objects.using(host_db)
         .filter(**kwargs)
         .select_related("featureset")
     )
-    return {fsl.slot: fsl.featureset for fsl in feature_set_links}
+    return {fsl.slot: fsl.featureset for fsl in links_feature_set}
 
 
 def get_label_links(
@@ -112,22 +112,15 @@ def get_label_links(
 def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
-    feature_set_links = host.feature_sets.through.objects.filter(**kwargs)
-    return feature_set_links
+    links_feature_set = host.feature_sets.through.objects.filter(**kwargs)
+    return links_feature_set
 
 
 def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
     link_model_name = link.__class__.__name__
-    if (
-        link_model_name == "ModelBase" or link_model_name == "RecordMeta"
-    ):  # we passed the type of the link
+    if link_model_name in {"Registry", "ModelBase"}:  # we passed the type of the link
         link_model_name = link.__name__
-    link_attr = link_model_name.replace(data.__class__.__name__, "")
-    if link_attr == "ExperimentalFactor":
-        link_attr = "experimental_factor"
-    else:
-        link_attr = link_attr.lower()
-    return link_attr
+    return link_model_name.replace(data.__class__.__name__, "").lower()
 
 
 # Custom aggregation for SQLite
@@ -182,14 +175,14 @@ def print_features(
     non_labels_msg = ""
     if self.id is not None and self.__class__ == Artifact or self.__class__ == Run:
         attr_name = "param" if print_params else "feature"
-        feature_values = (
-            getattr(self, f"{attr_name}_values")
+        _feature_values = (
+            getattr(self, f"_{attr_name}_values")
             .values(f"{attr_name}__name", f"{attr_name}__dtype")
             .annotate(values=custom_aggregate("value", self._state.db))
             .order_by(f"{attr_name}__name")
         )
-        if len(feature_values) > 0:
-            for fv in feature_values:
+        if len(_feature_values) > 0:
+            for fv in _feature_values:
                 feature_name = fv[f"{attr_name}__name"]
                 feature_dtype = fv[f"{attr_name}__dtype"]
                 values = fv["values"]
@@ -217,7 +210,7 @@ def print_features(
         for slot, feature_set in get_feature_set_by_slot_(self).items():
             features = feature_set.members
             # features.first() is a lot slower than features[0] here
-            name_field = get_default_str_field(features[0])
+            name_field = get_name_field(features[0])
             feature_names = list(features.values_list(name_field, flat=True)[:20])
             type_str = f": {feature_set.registry}" if print_types else ""
             feature_set_msg += (
@@ -246,7 +239,7 @@ def parse_feature_sets_from_anndata(
             from lamindb.core.storage._backed_access import backed_access
 
             using_key = settings._using_key
-            data_parse = backed_access(filepath, using_key)
+            data_parse = backed_access(filepath, using_key=using_key)
         else:
             data_parse = ad.read_h5ad(filepath, backed="r")
         type = "float"
@@ -316,13 +309,13 @@ def infer_feature_type_convert_json(
         if len(value) > 0:  # type: ignore
             first_element_type = type(next(iter(value)))
             if all(isinstance(elem, first_element_type) for elem in value):
-                if first_element_type == bool:
+                if first_element_type is bool:
                     return f"list[{FEATURE_TYPES['bool']}]", value
-                elif first_element_type == int:
+                elif first_element_type is int:
                     return f"list[{FEATURE_TYPES['int']}]", value
-                elif first_element_type == float:
+                elif first_element_type is float:
                     return f"list[{FEATURE_TYPES['float']}]", value
-                elif first_element_type == str:
+                elif first_element_type is str:
                     if str_as_ulabel:
                         return FEATURE_TYPES["str"] + "[ULabel]", value
                     else:
@@ -390,7 +383,7 @@ def filter(cls, **expression) -> QuerySet:
         feature = features.get(name=normalized_key)
         if not feature.dtype.startswith("cat"):
             feature_value = value_model.filter(feature=feature, value=value).one()
-            new_expression["feature_values"] = feature_value
+            new_expression["_feature_values"] = feature_value
         else:
             if isinstance(value, str):
                 label = ULabel.filter(name=value).one()
@@ -478,7 +471,7 @@ def _add_values(
     )
     # figure out which of the values go where
     features_labels = defaultdict(list)
-    feature_values = []
+    _feature_values = []
     not_validated_values = []
     for key, value in features_values.items():
         feature = model.filter(name=key).one()
@@ -508,7 +501,7 @@ def _add_values(
             feature_value = value_model.filter(**filter_kwargs).one_or_none()
             if feature_value is None:
                 feature_value = value_model(**filter_kwargs)
-            feature_values.append(feature_value)
+            _feature_values.append(feature_value)
         else:
             if isinstance(value, Record) or (
                 isinstance(value, Iterable) and isinstance(next(iter(value)), Record)
@@ -578,7 +571,7 @@ def _add_values(
             except Exception:
                 save(links, ignore_conflicts=True)
                 # now deal with links that were previously saved without a feature_id
-                saved_links = LinkORM.filter(
+                links_saved = LinkORM.filter(
                     **{
                         "artifact_id": self._host.id,
                         f"{field_name}__in": [
@@ -586,7 +579,7 @@ def _add_values(
                         ],
                     }
                 )
-                for link in saved_links.all():
+                for link in links_saved.all():
                     # TODO: also check for inconsistent features
                     if link.feature_id is None:
                         link.feature_id = [
@@ -595,13 +588,13 @@ def _add_values(
                             if l.id == getattr(link, field_name)
                         ][0]
                         link.save()
-    if feature_values:
-        save(feature_values)
+    if _feature_values:
+        save(_feature_values)
         if is_param:
-            LinkORM = self._host.param_values.through
+            LinkORM = self._host._param_values.through
             valuefield_id = "paramvalue_id"
         else:
-            LinkORM = self._host.feature_values.through
+            LinkORM = self._host._feature_values.through
             valuefield_id = "featurevalue_id"
         links = [
             LinkORM(
@@ -610,7 +603,7 @@ def _add_values(
                     valuefield_id: feature_value.id,
                 }
             )
-            for feature_value in feature_values
+            for feature_value in _feature_values
         ]
         # a link might already exist, to avoid raising a unique constraint
         # error, ignore_conflicts
@@ -683,10 +676,10 @@ def _add_set_from_df(
 ):
     """Add feature set corresponding to column names of DataFrame."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "DataFrame"  # noqa: S101
+        assert self._host._accessor == "DataFrame"  # noqa: S101
     else:
         # Collection
-        assert self._host.artifact.accessor == "DataFrame"  # noqa: S101
+        assert self._host.artifact._accessor == "DataFrame"  # noqa: S101
 
     # parse and register features
     registry = field.field.model
@@ -714,7 +707,7 @@ def _add_set_from_anndata(
 ):
     """Add features from AnnData."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "AnnData"  # noqa: S101
+        assert self._host._accessor == "AnnData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -744,16 +737,16 @@ def _add_set_from_mudata(
     if obs_fields is None:
         obs_fields = {}
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "MuData"  # noqa: S101
+        assert self._host._accessor == "MuData"  # noqa: S101
     else:
         raise NotImplementedError()
 
     # parse and register features
     mdata = self._host.load()
     feature_sets = {}
-    obs_features = features = Feature.from_values(mdata.obs.columns)
+    obs_features = Feature.from_values(mdata.obs.columns)
     if len(obs_features) > 0:
-        feature_sets["obs"] = FeatureSet(features=features)
+        feature_sets["obs"] = FeatureSet(features=obs_features)
     for modality, field in var_fields.items():
         modality_fs = parse_feature_sets_from_anndata(
             mdata[modality],
@@ -765,8 +758,20 @@ def _add_set_from_mudata(
         for k, v in modality_fs.items():
             feature_sets[f"['{modality}'].{k}"] = v
 
+    def unify_feature_sets_by_hash(feature_sets):
+        unique_values = {}
+
+        for key, value in feature_sets.items():
+            value_hash = value.hash  # Assuming each value has a .hash attribute
+            if value_hash in unique_values:
+                feature_sets[key] = unique_values[value_hash]
+            else:
+                unique_values[value_hash] = value
+
+        return feature_sets
+
     # link feature sets
-    self._host._feature_sets = feature_sets
+    self._host._feature_sets = unify_feature_sets_by_hash(feature_sets)
     self._host.save()
 
 
@@ -781,17 +786,12 @@ def _add_from(self, data: HasFeatures):
         registry = members[0].__class__
         # note here the features are transferred based on an unique field
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-        # TODO: get a default ID field for the registry
-        if hasattr(registry, "ontology_id"):
-            field = "ontology_id"
-        elif hasattr(registry, "ensembl_gene_id"):
-            field = "ensembl_gene_id"
-        elif hasattr(registry, "uniprotkb_id"):
-            field = "uniprotkb_id"
+        if hasattr(registry, "_ontology_id_field"):
+            field = registry._ontology_id_field
         # this will be e.g. be a list of ontology_ids or uids
         member_uids = list(members.values_list(field, flat=True))
         # create records from ontology_id
-        if field == "ontology_id" and len(member_uids) > 0:
+        if hasattr(registry, "_ontology_id_field") and len(member_uids) > 0:
             # create from bionty
             save(registry.from_values(member_uids, field=field))
         validated = registry.validate(member_uids, field=field, mute=True)
@@ -816,7 +816,7 @@ def _add_from(self, data: HasFeatures):
             member_uids, field=getattr(registry, field)
         )
         if feature_set_self is None:
-            if hasattr(registry, "organism"):
+            if hasattr(registry, "organism_id"):
                 logger.warning(
                     f"FeatureSet is not transferred, check if organism is set correctly: {feature_set}"
                 )

lamindb/core/_label_manager.py

@@ -4,13 +4,13 @@ from collections import defaultdict
 from typing import TYPE_CHECKING, Dict
 
 import numpy as np
-from lamin_utils import colors, logger
+from lamin_utils import colors
 from lnschema_core.models import Feature
 
 from lamindb._from_values import _print_values
 from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
-    get_default_str_field,
+    get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
 )
@@ -28,17 +28,19 @@ if TYPE_CHECKING:
 def get_labels_as_dict(self: HasFeatures, links: bool = False):
     exclude_set = {
         "feature_sets",
-        "unordered_artifacts",
-        "input_of",
+        "artifacts",
+        "input_of_runs",
         "collections",
-        "source_code_of",
+        "_source_code_artifact_of",
         "report_of",
         "environment_of",
-        "collection_links",
-        "artifact_links",
-        "feature_set_links",
+        "links_collection",
+        "links_artifact",
+        "links_feature_set",
         "previous_runs",
-        "feature_values",
+        "_feature_values",
+        "_lnschema_core_collection__actions_+",
+        "_actions",
     }
     labels = {}  # type: ignore
     if self.id is None:
@@ -62,7 +64,7 @@ def print_labels(self: HasFeatures, field: str = "name", print_types: bool = Fal
         try:
             labels_list = list(labels.values_list(field, flat=True))
             if len(labels_list) > 0:
-                get_default_str_field(labels)
+                get_name_field(labels)
                 print_values = _print_values(labels_list, n=10)
                 type_str = f": {related_model}" if print_types else ""
                 labels_msg += f"    .{related_name}{type_str} = {print_values}\n"
@@ -84,18 +86,14 @@ def validate_labels(labels: QuerySet | list | dict):
             return [], []
         registry = labels[0].__class__
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-        if hasattr(registry, "ontology_id"):
-            field = "ontology_id"
-        elif hasattr(registry, "ensembl_gene_id"):
-            field = "ensembl_gene_id"
-        elif hasattr(registry, "uniprotkb_id"):
-            field = "uniprotkb_id"
+        if hasattr(registry, "_ontology_id_field"):
+            field = registry._ontology_id_field
         # if the field value is None, use uid field
         label_uids = np.array(
             [getattr(label, field) for label in labels if label is not None]
         )
         # save labels from ontology_ids
-        if field == "ontology_id" and len(label_uids) > 0:
+        if hasattr(registry, "_ontology_id_field") and len(label_uids) > 0:
             try:
                 save(registry.from_values(label_uids, field=field))
             except Exception:  # noqa S110
@@ -201,10 +199,10 @@ class LabelManager:
                     transfer_fk_to_default_db_bulk(new_labels, using_key)
                 for label in labels:
                     # if the link table doesn't follow this convention, we'll ignore it
-                    if not hasattr(label, f"{data_name_lower}_links"):
+                    if not hasattr(label, f"links_{data_name_lower}"):
                         key = None
                     else:
-                        link = getattr(label, f"{data_name_lower}_links").get(
+                        link = getattr(label, f"links_{data_name_lower}").get(
                             **{f"{data_name_lower}_id": data.id}
                         )
                         if link.feature is not None:

lamindb/core/_mapped_collection.py

@@ -10,7 +10,7 @@ import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
 
-from .storage._backed_access import (
+from .storage._anndata_accessor import (
     ArrayType,
     ArrayTypes,
     GroupType,

lamindb/core/_run_context.py

@@ -11,7 +11,6 @@ from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 from lnschema_core import Run, Transform, ids
 from lnschema_core.models import Param, ParamValue, RunParamValue
-from lnschema_core.types import TransformType
 from lnschema_core.users import current_user_id
 
 from ._settings import settings
@@ -27,6 +26,7 @@ from .versioning import bump_version as bump_version_function
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
+    from lnschema_core.types import TransformType
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -279,14 +279,14 @@ class run_context:
                 ).one_or_none()
                 if is_run_from_ipython:
                     key, name = cls._track_notebook(path=path)
-                    transform_type = TransformType.notebook
+                    transform_type = "notebook"
                     transform_ref = None
                     transform_ref_type = None
                 else:
                     (name, key, transform_ref, transform_ref_type) = cls._track_script(
                         path=path
                     )
-                    transform_type = TransformType.script
+                    transform_type = "script"
                 # overwrite whatever is auto-detected in the notebook or script
                 if transform_settings.name is not None:
                     name = transform_settings.name
@@ -323,9 +323,7 @@ class run_context:
             cls.transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
-            new_run = (
-                False if cls.transform.type == TransformType.notebook.value else True
-            )  # type: ignore
+            new_run = False if cls.transform.type == "notebook" else True  # type: ignore
 
         run = None
         from lamindb._run import Run
@@ -479,7 +477,7 @@ class run_context:
                     transform.save()
                     logger.important(f"updated: {transform}")
             # check whether transform source code was already saved
-            if transform.source_code_id is not None:
+            if transform._source_code_artifact_id is not None:
                 response = None
                 if is_run_from_ipython:
                     if os.getenv("LAMIN_TESTING") is None:
@@ -491,7 +489,7 @@ class run_context:
                         response = "y"
                 else:
                     hash, _ = hash_file(cls.path)  # ignore hash_type for now
-                    if hash != transform.source_code.hash:
+                    if hash != transform._source_code_artifact.hash:
                         # only if hashes don't match, we need user input
                         if os.getenv("LAMIN_TESTING") is None:
                             response = input(

lamindb/core/datasets/_core.py

@@ -83,7 +83,7 @@ def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts(
         ln.settings.verbosity = "error"
         ln.Feature(name="assay", dtype=[bt.ExperimentalFactor]).save()
         ln.Feature(name="organism", dtype=[bt.Organism]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008896").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008896").save()
         ln.settings.verbosity = verbosity
 
     return Path(filepath)
@@ -186,16 +186,16 @@ def anndata_mouse_sc_lymph_node(
         verbosity = ln.settings.verbosity
         ln.settings.verbosity = "error"
         # strain
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0004472").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0004472").save()
         # developmental stage
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0001272").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0001272").save()
         # tissue
-        bt.Tissue.from_public(ontology_id="UBERON:0001542").save()
+        bt.Tissue.from_source(ontology_id="UBERON:0001542").save()
         # cell types
         ln.save(bt.CellType.from_values(["CL:0000115", "CL:0000738"], "ontology_id"))
         # assays
         ln.Feature(name="assay", dtype=[bt.ExperimentalFactor]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008913").save()
         # genes
         validated = bt.Gene.public(organism="mouse").validate(
             adata.var.index, field="ensembl_gene_id"
@@ -323,7 +323,7 @@ def anndata_human_immune_cells(
         ln.Feature(name="tissue", dtype=[bt.Tissue]).save()
         ln.Feature(name="organism", dtype=[bt.Organism]).save()
         ln.Feature(name="donor", dtype=[ln.ULabel]).save()
-        bt.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
+        bt.ExperimentalFactor.from_source(ontology_id="EFO:0008913").save()
         ln.save([ln.ULabel(name=name) for name in adata.obs.donor.unique()])
         ln.settings.verbosity = verbosity
     return adata
@@ -332,7 +332,7 @@ def anndata_human_immune_cells(
 def anndata_with_obs() -> ad.AnnData:
     """Create a mini anndata with cell_type, disease and tissue."""
     import anndata as ad
-    import bionty_base
+    import bionty.base as bionty_base
 
     celltypes = ["T cell", "hematopoietic stem cell", "hepatocyte", "my new cell type"]
     celltype_ids = ["CL:0000084", "CL:0000037", "CL:0000182", ""]

lamindb/core/exceptions.py

@@ -10,6 +10,7 @@ The registry base class:
    NoTitleError
    MissingTransformSettings
    UpdateTransformSettings
+   IntegrityError
 
 """
 
@@ -25,6 +26,16 @@ class ValidationError(SystemExit):
 # -------------------------------------------------------------------------------------
 
 
+class IntegrityError(Exception):
+    """Integrity error.
+
+    For instance, it's not allowed to delete artifacts outside managed storage
+    locations.
+    """
+
+    pass
+
+
 class NotebookNotSavedError(Exception):
     """Notebook wasn't saved."""
 

lamindb/core/schema.py

@@ -4,16 +4,16 @@ from django.db.models import ManyToManyField
 from lnschema_core.models import Feature, FeatureSet, LinkORM, Record
 
 
-def dict_schema_name_to_model_name(orm: type[Record]) -> dict[str, Record]:
+def dict_schema_name_to_model_name(registry: type[Record]) -> dict[str, Record]:
     d: dict = {
         i.related_model.__get_name_with_schema__(): i.related_model
-        for i in orm._meta.related_objects
+        for i in registry._meta.related_objects
         if i.related_name is not None
     }
     d.update(
         {
             i.related_model.__get_name_with_schema__(): i.related_model
-            for i in orm._meta.many_to_many
+            for i in registry._meta.many_to_many
             if i.name is not None
         }
     )
@@ -21,12 +21,12 @@ def dict_schema_name_to_model_name(orm: type[Record]) -> dict[str, Record]:
 
 
 def dict_related_model_to_related_name(
-    orm: type[Record], links: bool = False
+    registry: type[Record], links: bool = False
 ) -> dict[str, str]:
     def include(model: Record):
         return not links != issubclass(model, LinkORM)
 
-    related_objects = orm._meta.related_objects + orm._meta.many_to_many
+    related_objects = registry._meta.related_objects + registry._meta.many_to_many
     d: dict = {
         record.related_model.__get_name_with_schema__(): (
             record.related_name

lamindb/core/storage/__init__.py

@@ -1,4 +1,13 @@
-"""Storage tools.
+"""Storage API.
+
+Valid suffixes.
+
+.. autosummary::
+   :toctree: .
+
+   VALID_SUFFIXES
+
+Array accessors.
 
 .. autosummary::
    :toctree: .
@@ -6,10 +15,11 @@
    AnnDataAccessor
    BackedAccessor
 """
+
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
 from ._anndata_sizes import size_adata
 from ._backed_access import AnnDataAccessor, BackedAccessor
-from ._valid_suffixes import VALID_COMPOSITE_SUFFIXES, VALID_SUFFIXES
+from ._valid_suffixes import VALID_SUFFIXES
 from .objects import infer_suffix, write_to_disk
 from .paths import delete_storage, load_to_memory