lamindb 0.74.3__py3-none-any.whl → 0.75.0__py3-none-any.whl

lamindb/_is_versioned.py

@@ -16,7 +16,7 @@ def _add_to_version_family(
 ):
     old_uid = self.uid
     new_uid, version = get_uid_from_old_version(is_new_version_of, version)
-    if self.__class__.__name__ == "Artifact" and self.key_is_virtual:
+    if self.__class__.__name__ == "Artifact" and self._key_is_virtual:
         old_path = self.path
         new_path = get_new_path_from_uid(
             old_path=old_path, old_uid=old_uid, new_uid=new_uid

lamindb/_parents.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import builtins
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
@@ -10,7 +10,7 @@ from lnschema_core.models import HasParents, format_field_value
 
 from lamindb._utils import attach_func_to_class_method
 
-from ._record import get_default_str_field
+from ._record import get_name_field
 
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
@@ -61,7 +61,7 @@ def view_parents(
     distance: int = 5,
 ):
     if field is None:
-        field = get_default_str_field(self)
+        field = get_name_field(self)
     if not isinstance(field, str):
         field = field.field.name
 
@@ -137,10 +137,14 @@ def view_lineage(data: Artifact | Collection, with_children: bool = True) -> Non
 
 
 def _view_parents(
-    record: Record, field: str, with_children: bool = False, distance: int = 100
+    record: Record,
+    field: str,
+    with_children: bool = False,
+    distance: int = 100,
+    attr_name: Literal["parents", "predecessors"] = "parents",
 ):
     """Graph of parents."""
-    if not hasattr(record, "parents"):
+    if not hasattr(record, attr_name):
         raise NotImplementedError(
             f"Parents view is not supported for {record.__class__.__name__}!"
         )
@@ -149,13 +153,17 @@ def _view_parents(
 
     df_edges = None
     df_edges_parents = _df_edges_from_parents(
-        record=record, field=field, distance=distance
+        record=record, field=field, distance=distance, attr_name=attr_name
     )
     if df_edges_parents is not None:
         df_edges = df_edges_parents
     if with_children:
         df_edges_children = _df_edges_from_parents(
-            record=record, field=field, distance=distance, children=True
+            record=record,
+            field=field,
+            distance=distance,
+            children=True,
+            attr_name=attr_name,
         )
         if df_edges_children is not None:
             if df_edges is not None:
@@ -197,12 +205,18 @@ def _view_parents(
     _view(u)
 
 
-def _get_parents(record: Record, field: str, distance: int, children: bool = False):
+def _get_parents(
+    record: Record,
+    field: str,
+    distance: int,
+    children: bool = False,
+    attr_name: Literal["parents", "predecessors"] = "parents",
+):
     """Recursively get parent records within a distance."""
     if children:
-        key = "parents"
+        key = attr_name
     else:
-        key = "children"
+        key = "children" if attr_name == "parents" else "successors"  # type: ignore
     model = record.__class__
     condition = f"{key}__{field}"
     results = model.filter(**{condition: record.__getattribute__(field)}).all()
@@ -228,12 +242,23 @@ def _get_parents(record: Record, field: str, distance: int, children: bool = Fal
 
 
 def _df_edges_from_parents(
-    record: Record, field: str, distance: int, children: bool = False
+    record: Record,
+    field: str,
+    distance: int,
+    children: bool = False,
+    attr_name: Literal["parents", "predecessors"] = "parents",
 ):
     """Construct a DataFrame of edges as the input of graphviz.Digraph."""
-    key = "children" if children else "parents"
+    if attr_name == "parents":
+        key = "children" if children else "parents"
+    else:
+        key = "successors" if children else "predecessors"
     parents = _get_parents(
-        record=record, field=field, distance=distance, children=children
+        record=record,
+        field=field,
+        distance=distance,
+        children=children,
+        attr_name=attr_name,
     )
     all = record.__class__.objects
     records = parents | all.filter(id=record.id)

lamindb/_record.py

@@ -160,19 +160,22 @@ def from_values(
     field: StrField | None = None,
     create: bool = False,
     organism: Record | str | None = None,
-    public_source: Record | None = None,
+    source: Record | None = None,
     mute: bool = False,
 ) -> list[Record]:
     """{}"""  # noqa: D415
-    from_public = True if cls.__module__.startswith("lnschema_bionty.") else False
-    field_str = get_default_str_field(cls, field=field)
+    from_source = True if cls.__module__.startswith("bionty.") else False
+    # if records from source is already saved in db, skip from_source
+    if isinstance(source, Record) and source.in_db:
+        from_source = False
+    field_str = get_name_field(cls, field=field)
     return get_or_create_records(
         iterable=values,
         field=getattr(cls, field_str),
         create=create,
-        from_public=from_public,
+        from_source=from_source,
         organism=organism,
-        public_source=public_source,
+        source=source,
         mute=mute,
     )
 
@@ -284,7 +287,7 @@ def _lookup(
 ) -> NamedTuple:
     """{}"""  # noqa: D415
     queryset = _queryset(cls, using_key=using_key)
-    field = get_default_str_field(orm=queryset.model, field=field)
+    field = get_name_field(orm=queryset.model, field=field)
 
     return Lookup(
         records=queryset,
@@ -293,7 +296,7 @@ def _lookup(
         prefix="ln",
     ).lookup(
         return_field=(
-            get_default_str_field(orm=queryset.model, field=return_field)
+            get_name_field(orm=queryset.model, field=return_field)
             if return_field is not None
             else None
         )
@@ -311,7 +314,7 @@ def lookup(
     return _lookup(cls=cls, field=field, return_field=return_field)
 
 
-def get_default_str_field(
+def get_name_field(
     orm: Record | QuerySet | Manager,
     *,
     field: str | StrField | None = None,
@@ -321,14 +324,11 @@ def get_default_str_field(
         orm = orm.model
     model_field_names = [i.name for i in orm._meta.fields]
 
-    # set default field
+    # set to default name field
     if field is None:
-        if orm._meta.model.__name__ == "Run":
-            field = orm._meta.get_field("created_at")
-        elif orm._meta.model.__name__ == "User":
-            field = orm._meta.get_field("handle")
+        if hasattr(orm, "_name_field"):
+            field = orm._meta.get_field(orm._name_field)
         elif "name" in model_field_names:
-            # by default use the name field
             field = orm._meta.get_field("name")
         else:
             # first char or text field that doesn't contain "id"
@@ -339,7 +339,7 @@ def get_default_str_field(
                     field = i
                     break
 
-        # no default field can be found
+        # no default name field can be found
         if field is None:
             raise ValueError(
                 "please pass a Record string field, e.g., `CellType.name`!"
@@ -443,9 +443,8 @@ def update_fk_to_default_db(
 
 FKBULK = [
     "organism",
-    "public_source",
-    "latest_report",  # Transform
-    "source_code",  # Transform
+    "source",
+    "_source_code_artifact",  # Transform
     "report",  # Run
 ]
 
@@ -523,7 +522,7 @@ def save(self, *args, **kwargs) -> Record:
     artifacts: list = []
     if self.__class__.__name__ == "Collection" and self.id is not None:
         # when creating a new collection without being able to access artifacts
-        artifacts = self.artifacts.list()
+        artifacts = self.ordered_artifacts.list()
     # transfer of the record to the default db with fk fields
     result = transfer_to_default_db(self, using_key)
     if result is not None:
@@ -538,7 +537,7 @@ def save(self, *args, **kwargs) -> Record:
                 logger.info("transfer artifacts")
                 for artifact in artifacts:
                     artifact.save()
-                self.unordered_artifacts.add(*artifacts)
+                self.artifacts.add(*artifacts)
         if hasattr(self, "labels"):
             from copy import copy
 

lamindb/_save.py

@@ -85,9 +85,9 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
             r for r in non_artifacts_new if hasattr(r, "_parents")
         ]
         if len(non_artifacts_with_parents) > 0:
-            # this can only happen within lnschema_bionty right now!!
+            # this can only happen within bionty right now!!
             # we might extend to core lamindb later
-            from lnschema_bionty.core import add_ontology
+            from bionty.core import add_ontology
 
             add_ontology(non_artifacts_with_parents)
 

lamindb/_transform.py

@@ -1,11 +1,17 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Run, Transform
-from lnschema_core.types import TransformType
 
+from ._parents import _view_parents
 from ._run import delete_run_artifacts
 from .core.versioning import process_is_new_version_of
 
+if TYPE_CHECKING:
+    from lnschema_core.types import TransformType
+
 
 def __init__(transform: Transform, *args, **kwargs):
     if len(args) == len(transform._meta.concrete_fields):
@@ -18,9 +24,7 @@ def __init__(transform: Transform, *args, **kwargs):
     )
     (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
-    type: TransformType | None = (
-        kwargs.pop("type") if "type" in kwargs else TransformType.pipeline
-    )
+    type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
     reference_type: str | None = (
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
@@ -55,19 +59,13 @@ def __init__(transform: Transform, *args, **kwargs):
 
 
 def delete(self) -> None:
-    # set latest_report to None, it's tracked through the latest run
-    latest_report = None
-    if self.latest_report is not None:
-        latest_report = self.latest_report
-        self.latest_report = None
-    source_code = None
-    if self.source_code is not None:
-        source_code = self.source_code
-        self.source_code = None
-    if latest_report is not None or source_code is not None:
+    _source_code_artifact = None
+    if self._source_code_artifact is not None:
+        _source_code_artifact = self._source_code_artifact
+        self._source_code_artifact = None
         self.save()
-    if source_code is not None:
-        source_code.delete(permanent=True)
+    if _source_code_artifact is not None:
+        _source_code_artifact.delete(permanent=True)
     # query all runs and delete their artifacts
     runs = Run.filter(transform=self)
     for run in runs:
@@ -78,10 +76,23 @@ def delete(self) -> None:
 
 
 @property  # type: ignore
+@doc_args(Transform.latest_run.__doc__)
 def latest_run(self) -> Run:
+    """{}"""  # noqa: D415
     return self.runs.order_by("-started_at").first()
 
 
+def view_lineage(self, with_successors: bool = False, distance: int = 5):
+    return _view_parents(
+        record=self,
+        field="name",
+        with_children=with_successors,
+        distance=distance,
+        attr_name="predecessors",
+    )
+
+
 Transform.__init__ = __init__
 Transform.delete = delete
 Transform.latest_run = latest_run
+Transform.view_lineage = view_lineage

lamindb/core/_data.py

@@ -20,7 +20,7 @@ from lnschema_core.models import (
 
 from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
-from lamindb._record import get_default_str_field
+from lamindb._record import get_name_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
@@ -129,31 +129,29 @@ def describe(self: HasFeatures, print_types: bool = False):
         # prefetch m-2-m relationships
         self = (
             self.__class__.objects.using(self._state.db)
-            .prefetch_related("feature_sets", "input_of")
+            .prefetch_related("feature_sets", "input_of_runs")
             .get(id=self.id)
         )
 
     # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
         fields_values = [(field, getattr(self, field)) for field in foreign_key_fields]
-        type_str = (
-            lambda attr: f": {attr.__class__.__get_name_with_schema__()}"
-            if print_types
-            else ""
+        type_str = lambda attr: (
+            f": {attr.__class__.__get_name_with_schema__()}" if print_types else ""
         )
         related_msg = "".join(
             [
-                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_default_str_field(attr)))}\n"
+                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_name_field(attr)))}\n"
                 for (field_name, attr) in fields_values
                 if attr is not None
             ]
         )
         prov_msg += related_msg
     # input of
-    if self.id is not None and self.input_of.exists():
-        values = [format_field_value(i.started_at) for i in self.input_of.all()]
+    if self.id is not None and self.input_of_runs.exists():
+        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
         type_str = ": Run" if print_types else ""  # type: ignore
-        prov_msg += f"    .input_of{type_str} = {values}\n"
+        prov_msg += f"    .input_of_runs{type_str} = {values}\n"
     if prov_msg:
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
@@ -210,11 +208,11 @@ def get_labels(
             ).all()
     if flat_names:
         # returns a flat list of names
-        from lamindb._record import get_default_str_field
+        from lamindb._record import get_name_field
 
         values = []
         for v in qs_by_registry.values():
-            values += v.list(get_default_str_field(v))
+            values += v.list(get_name_field(v))
         return values
     if len(registries_to_check) == 1 and registry in qs_by_registry:
         return qs_by_registry[registry]
@@ -304,12 +302,12 @@ def add_labels(
             if len(linked_labels) > 0:
                 labels_accessor.remove(*linked_labels)
             labels_accessor.add(*records, through_defaults={"feature_id": feature.id})
-        feature_set_links = get_feature_set_links(self)
-        feature_set_ids = [link.featureset_id for link in feature_set_links.all()]
+        links_feature_set = get_feature_set_links(self)
+        feature_set_ids = [link.featureset_id for link in links_feature_set.all()]
         # get all linked features of type Feature
         feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
         {
-            feature_set_links.filter(featureset_id=feature_set.id)
+            links_feature_set.filter(featureset_id=feature_set.id)
             .one()
             .slot: feature_set.features.all()
             for feature_set in feature_sets
@@ -415,7 +413,7 @@ def _track_run_input(
         # generalize below for more than one data batch
         if len(input_data) == 1:
             if input_data[0].transform is not None:
-                run.transform.parents.add(input_data[0].transform)
+                run.transform.predecessors.add(input_data[0].transform)
 
 
 HasFeatures.describe = describe

lamindb/core/_feature_manager.py

@@ -39,7 +39,7 @@ from lamindb._feature import FEATURE_TYPES, convert_numpy_dtype_to_lamin_feature
 from lamindb._feature_set import DICT_KEYS_TYPE, FeatureSet
 from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
-    get_default_str_field,
+    get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
 )
@@ -88,12 +88,12 @@ def get_feature_set_by_slot_(host) -> dict:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
     # otherwise, we need a query
-    feature_set_links = (
+    links_feature_set = (
         host.feature_sets.through.objects.using(host_db)
         .filter(**kwargs)
         .select_related("featureset")
     )
-    return {fsl.slot: fsl.featureset for fsl in feature_set_links}
+    return {fsl.slot: fsl.featureset for fsl in links_feature_set}
 
 
 def get_label_links(
@@ -112,8 +112,8 @@ def get_label_links(
 def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     host_id_field = get_host_id_field(host)
     kwargs = {host_id_field: host.id}
-    feature_set_links = host.feature_sets.through.objects.filter(**kwargs)
-    return feature_set_links
+    links_feature_set = host.feature_sets.through.objects.filter(**kwargs)
+    return links_feature_set
 
 
 def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
@@ -122,12 +122,7 @@ def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
         link_model_name == "ModelBase" or link_model_name == "RecordMeta"
     ):  # we passed the type of the link
         link_model_name = link.__name__
-    link_attr = link_model_name.replace(data.__class__.__name__, "")
-    if link_attr == "ExperimentalFactor":
-        link_attr = "experimental_factor"
-    else:
-        link_attr = link_attr.lower()
-    return link_attr
+    return link_model_name.replace(data.__class__.__name__, "").lower()
 
 
 # Custom aggregation for SQLite
@@ -182,14 +177,14 @@ def print_features(
     non_labels_msg = ""
     if self.id is not None and self.__class__ == Artifact or self.__class__ == Run:
         attr_name = "param" if print_params else "feature"
-        feature_values = (
-            getattr(self, f"{attr_name}_values")
+        _feature_values = (
+            getattr(self, f"_{attr_name}_values")
             .values(f"{attr_name}__name", f"{attr_name}__dtype")
             .annotate(values=custom_aggregate("value", self._state.db))
             .order_by(f"{attr_name}__name")
         )
-        if len(feature_values) > 0:
-            for fv in feature_values:
+        if len(_feature_values) > 0:
+            for fv in _feature_values:
                 feature_name = fv[f"{attr_name}__name"]
                 feature_dtype = fv[f"{attr_name}__dtype"]
                 values = fv["values"]
@@ -217,7 +212,7 @@ def print_features(
         for slot, feature_set in get_feature_set_by_slot_(self).items():
             features = feature_set.members
             # features.first() is a lot slower than features[0] here
-            name_field = get_default_str_field(features[0])
+            name_field = get_name_field(features[0])
             feature_names = list(features.values_list(name_field, flat=True)[:20])
             type_str = f": {feature_set.registry}" if print_types else ""
             feature_set_msg += (
@@ -246,7 +241,7 @@ def parse_feature_sets_from_anndata(
             from lamindb.core.storage._backed_access import backed_access
 
             using_key = settings._using_key
-            data_parse = backed_access(filepath, using_key)
+            data_parse = backed_access(filepath, using_key=using_key)
         else:
             data_parse = ad.read_h5ad(filepath, backed="r")
         type = "float"
@@ -316,13 +311,13 @@ def infer_feature_type_convert_json(
         if len(value) > 0:  # type: ignore
             first_element_type = type(next(iter(value)))
             if all(isinstance(elem, first_element_type) for elem in value):
-                if first_element_type == bool:
+                if first_element_type is bool:
                     return f"list[{FEATURE_TYPES['bool']}]", value
-                elif first_element_type == int:
+                elif first_element_type is int:
                     return f"list[{FEATURE_TYPES['int']}]", value
-                elif first_element_type == float:
+                elif first_element_type is float:
                     return f"list[{FEATURE_TYPES['float']}]", value
-                elif first_element_type == str:
+                elif first_element_type is str:
                     if str_as_ulabel:
                         return FEATURE_TYPES["str"] + "[ULabel]", value
                     else:
@@ -390,7 +385,7 @@ def filter(cls, **expression) -> QuerySet:
         feature = features.get(name=normalized_key)
         if not feature.dtype.startswith("cat"):
             feature_value = value_model.filter(feature=feature, value=value).one()
-            new_expression["feature_values"] = feature_value
+            new_expression["_feature_values"] = feature_value
         else:
             if isinstance(value, str):
                 label = ULabel.filter(name=value).one()
@@ -478,7 +473,7 @@ def _add_values(
     )
     # figure out which of the values go where
     features_labels = defaultdict(list)
-    feature_values = []
+    _feature_values = []
     not_validated_values = []
     for key, value in features_values.items():
         feature = model.filter(name=key).one()
@@ -508,7 +503,7 @@ def _add_values(
             feature_value = value_model.filter(**filter_kwargs).one_or_none()
             if feature_value is None:
                 feature_value = value_model(**filter_kwargs)
-            feature_values.append(feature_value)
+            _feature_values.append(feature_value)
         else:
             if isinstance(value, Record) or (
                 isinstance(value, Iterable) and isinstance(next(iter(value)), Record)
@@ -578,7 +573,7 @@ def _add_values(
             except Exception:
                 save(links, ignore_conflicts=True)
                 # now deal with links that were previously saved without a feature_id
-                saved_links = LinkORM.filter(
+                links_saved = LinkORM.filter(
                     **{
                         "artifact_id": self._host.id,
                         f"{field_name}__in": [
@@ -586,7 +581,7 @@ def _add_values(
                         ],
                     }
                 )
-                for link in saved_links.all():
+                for link in links_saved.all():
                     # TODO: also check for inconsistent features
                     if link.feature_id is None:
                         link.feature_id = [
@@ -595,13 +590,13 @@ def _add_values(
                             if l.id == getattr(link, field_name)
                         ][0]
                         link.save()
-    if feature_values:
-        save(feature_values)
+    if _feature_values:
+        save(_feature_values)
         if is_param:
-            LinkORM = self._host.param_values.through
+            LinkORM = self._host._param_values.through
             valuefield_id = "paramvalue_id"
         else:
-            LinkORM = self._host.feature_values.through
+            LinkORM = self._host._feature_values.through
             valuefield_id = "featurevalue_id"
         links = [
             LinkORM(
@@ -610,7 +605,7 @@ def _add_values(
                     valuefield_id: feature_value.id,
                 }
             )
-            for feature_value in feature_values
+            for feature_value in _feature_values
         ]
         # a link might already exist, to avoid raising a unique constraint
         # error, ignore_conflicts
@@ -683,10 +678,10 @@ def _add_set_from_df(
 ):
     """Add feature set corresponding to column names of DataFrame."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "DataFrame"  # noqa: S101
+        assert self._host._accessor == "DataFrame"  # noqa: S101
     else:
         # Collection
-        assert self._host.artifact.accessor == "DataFrame"  # noqa: S101
+        assert self._host.artifact._accessor == "DataFrame"  # noqa: S101
 
     # parse and register features
     registry = field.field.model
@@ -714,7 +709,7 @@ def _add_set_from_anndata(
 ):
     """Add features from AnnData."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "AnnData"  # noqa: S101
+        assert self._host._accessor == "AnnData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -744,7 +739,7 @@ def _add_set_from_mudata(
     if obs_fields is None:
         obs_fields = {}
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "MuData"  # noqa: S101
+        assert self._host._accessor == "MuData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -781,17 +776,12 @@ def _add_from(self, data: HasFeatures):
         registry = members[0].__class__
         # note here the features are transferred based on an unique field
         field = REGISTRY_UNIQUE_FIELD.get(registry.__name__.lower(), "uid")
-        # TODO: get a default ID field for the registry
-        if hasattr(registry, "ontology_id"):
-            field = "ontology_id"
-        elif hasattr(registry, "ensembl_gene_id"):
-            field = "ensembl_gene_id"
-        elif hasattr(registry, "uniprotkb_id"):
-            field = "uniprotkb_id"
+        if hasattr(registry, "_ontology_id_field"):
+            field = registry._ontology_id_field
         # this will be e.g. be a list of ontology_ids or uids
         member_uids = list(members.values_list(field, flat=True))
         # create records from ontology_id
-        if field == "ontology_id" and len(member_uids) > 0:
+        if hasattr(registry, "_ontology_id_field") and len(member_uids) > 0:
             # create from bionty
             save(registry.from_values(member_uids, field=field))
         validated = registry.validate(member_uids, field=field, mute=True)
@@ -816,7 +806,7 @@ def _add_from(self, data: HasFeatures):
             member_uids, field=getattr(registry, field)
         )
         if feature_set_self is None:
-            if hasattr(registry, "organism"):
+            if hasattr(registry, "organism_id"):
                 logger.warning(
                     f"FeatureSet is not transferred, check if organism is set correctly: {feature_set}"
                 )