lamindb 0.74.3__py3-none-any.whl → 0.75.1__py3-none-any.whl

lamindb/_record.py

@@ -36,9 +36,9 @@ def init_self_from_db(self: Record, existing_record: Record):
     self._state.db = "default"
 
 
-def validate_required_fields(orm: Record, kwargs):
+def validate_required_fields(record: Record, kwargs):
     required_fields = {
-        k.name for k in orm._meta.fields if not k.null and k.default is None
+        k.name for k in record._meta.fields if not k.null and k.default is None
     }
     required_fields_not_passed = {k: None for k in required_fields if k not in kwargs}
     kwargs.update(required_fields_not_passed)
@@ -77,9 +77,9 @@ def suggest_records_with_similar_names(record: Record, kwargs) -> bool:
     return False
 
 
-def __init__(orm: Record, *args, **kwargs):
+def __init__(record: Record, *args, **kwargs):
     if not args:
-        validate_required_fields(orm, kwargs)
+        validate_required_fields(record, kwargs)
 
         # do not search for names if an id is passed; this is important
         # e.g. when synching ids from the notebook store to lamindb
@@ -87,29 +87,29 @@ def __init__(orm: Record, *args, **kwargs):
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
         if settings.creation.search_names and not has_consciously_provided_uid:
-            match = suggest_records_with_similar_names(orm, kwargs)
+            match = suggest_records_with_similar_names(record, kwargs)
             if match:
                 if "version" in kwargs:
                     version_comment = " and version"
-                    existing_record = orm.__class__.filter(
+                    existing_record = record.__class__.filter(
                         name=kwargs["name"], version=kwargs["version"]
                     ).one_or_none()
                 else:
                     version_comment = ""
-                    existing_record = orm.__class__.filter(name=kwargs["name"]).one()
+                    existing_record = record.__class__.filter(name=kwargs["name"]).one()
                 if existing_record is not None:
                     logger.important(
-                        f"returning existing {orm.__class__.__name__} record with same"
+                        f"returning existing {record.__class__.__name__} record with same"
                         f" name{version_comment}: '{kwargs['name']}'"
                     )
-                    init_self_from_db(orm, existing_record)
+                    init_self_from_db(record, existing_record)
                     return None
-        super(Record, orm).__init__(**kwargs)
-    elif len(args) != len(orm._meta.concrete_fields):
+        super(Record, record).__init__(**kwargs)
+    elif len(args) != len(record._meta.concrete_fields):
         raise ValueError("please provide keyword arguments, not plain arguments")
     else:
         # object is loaded from DB (**kwargs could be omitted below, I believe)
-        super(Record, orm).__init__(*args, **kwargs)
+        super(Record, record).__init__(*args, **kwargs)
 
 
 @classmethod  # type:ignore
@@ -160,19 +160,22 @@ def from_values(
     field: StrField | None = None,
     create: bool = False,
     organism: Record | str | None = None,
-    public_source: Record | None = None,
+    source: Record | None = None,
     mute: bool = False,
 ) -> list[Record]:
     """{}"""  # noqa: D415
-    from_public = True if cls.__module__.startswith("lnschema_bionty.") else False
-    field_str = get_default_str_field(cls, field=field)
+    from_source = True if cls.__module__.startswith("bionty.") else False
+    # if records from source is already saved in db, skip from_source
+    if isinstance(source, Record) and source.in_db:
+        from_source = False
+    field_str = get_name_field(cls, field=field)
     return get_or_create_records(
         iterable=values,
         field=getattr(cls, field_str),
         create=create,
-        from_public=from_public,
+        from_source=from_source,
         organism=organism,
-        public_source=public_source,
+        source=source,
         mute=mute,
     )
 
@@ -188,11 +191,11 @@ def _search(
     truncate_words: bool = False,
 ) -> QuerySet:
     input_queryset = _queryset(cls, using_key=using_key)
-    orm = input_queryset.model
+    registry = input_queryset.model
     if field is None:
         fields = [
             field.name
-            for field in orm._meta.fields
+            for field in registry._meta.fields
             if field.get_internal_type() in {"CharField", "TextField"}
         ]
     else:
@@ -284,7 +287,7 @@ def _lookup(
 ) -> NamedTuple:
     """{}"""  # noqa: D415
     queryset = _queryset(cls, using_key=using_key)
-    field = get_default_str_field(orm=queryset.model, field=field)
+    field = get_name_field(registry=queryset.model, field=field)
 
     return Lookup(
         records=queryset,
@@ -293,7 +296,7 @@ def _lookup(
         prefix="ln",
     ).lookup(
         return_field=(
-            get_default_str_field(orm=queryset.model, field=return_field)
+            get_name_field(registry=queryset.model, field=return_field)
             if return_field is not None
             else None
         )
@@ -311,35 +314,32 @@ def lookup(
     return _lookup(cls=cls, field=field, return_field=return_field)
 
 
-def get_default_str_field(
-    orm: Record | QuerySet | Manager,
+def get_name_field(
+    registry: type[Record] | QuerySet | Manager,
     *,
     field: str | StrField | None = None,
 ) -> str:
-    """Get the 1st char or text field from the orm."""
-    if isinstance(orm, (QuerySet, Manager)):
-        orm = orm.model
-    model_field_names = [i.name for i in orm._meta.fields]
+    """Get the 1st char or text field from the registry."""
+    if isinstance(registry, (QuerySet, Manager)):
+        registry = registry.model
+    model_field_names = [i.name for i in registry._meta.fields]
 
-    # set default field
+    # set to default name field
     if field is None:
-        if orm._meta.model.__name__ == "Run":
-            field = orm._meta.get_field("created_at")
-        elif orm._meta.model.__name__ == "User":
-            field = orm._meta.get_field("handle")
+        if hasattr(registry, "_name_field"):
+            field = registry._meta.get_field(registry._name_field)
         elif "name" in model_field_names:
-            # by default use the name field
-            field = orm._meta.get_field("name")
+            field = registry._meta.get_field("name")
         else:
             # first char or text field that doesn't contain "id"
-            for i in orm._meta.fields:
+            for i in registry._meta.fields:
                 if "id" in i.name:
                     continue
                 if i.get_internal_type() in {"CharField", "TextField"}:
                     field = i
                     break
 
-        # no default field can be found
+        # no default name field can be found
         if field is None:
             raise ValueError(
                 "please pass a Record string field, e.g., `CellType.name`!"
@@ -360,7 +360,7 @@ def get_default_str_field(
 def _queryset(cls: Record | QuerySet | Manager, using_key: str) -> QuerySet:
     if isinstance(cls, (QuerySet, Manager)):
         return cls.all()
-    elif using_key is None:
+    elif using_key is None or using_key == "default":
         return cls.objects.all()
     else:
         # using must be called on cls, otherwise the connection isn't found
@@ -443,9 +443,8 @@ def update_fk_to_default_db(
 
 FKBULK = [
     "organism",
-    "public_source",
-    "latest_report",  # Transform
-    "source_code",  # Transform
+    "source",
+    "_source_code_artifact",  # Transform
     "report",  # Run
 ]
 
@@ -523,7 +522,7 @@ def save(self, *args, **kwargs) -> Record:
     artifacts: list = []
     if self.__class__.__name__ == "Collection" and self.id is not None:
         # when creating a new collection without being able to access artifacts
-        artifacts = self.artifacts.list()
+        artifacts = self.ordered_artifacts.list()
     # transfer of the record to the default db with fk fields
     result = transfer_to_default_db(self, using_key)
     if result is not None:
@@ -538,7 +537,7 @@ def save(self, *args, **kwargs) -> Record:
                 logger.info("transfer artifacts")
                 for artifact in artifacts:
                     artifact.save()
-                self.unordered_artifacts.add(*artifacts)
+                self.artifacts.add(*artifacts)
         if hasattr(self, "labels"):
             from copy import copy
 

lamindb/_save.py

@@ -85,9 +85,9 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
             r for r in non_artifacts_new if hasattr(r, "_parents")
         ]
         if len(non_artifacts_with_parents) > 0:
-            # this can only happen within lnschema_bionty right now!!
+            # this can only happen within bionty right now!!
             # we might extend to core lamindb later
-            from lnschema_bionty.core import add_ontology
+            from bionty.core import add_ontology
 
             add_ontology(non_artifacts_with_parents)
 
@@ -108,21 +108,21 @@ def bulk_create(records: Iterable[Record], ignore_conflicts: bool | None = False
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
-    for orm, records in records_by_orm.items():
-        orm.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
+    for registry, records in records_by_orm.items():
+        registry.objects.bulk_create(records, ignore_conflicts=ignore_conflicts)
 
 
 def bulk_update(records: Iterable[Record], ignore_conflicts: bool | None = False):
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
-    for orm, records in records_by_orm.items():
+    for registry, records in records_by_orm.items():
         field_names = [
             field.name
-            for field in orm._meta.fields
+            for field in registry._meta.fields
             if (field.name != "created_at" and field.name != "id")
         ]
-        orm.objects.bulk_update(records, field_names)
+        registry.objects.bulk_update(records, field_names)
 
 
 # This is also used within Artifact.save()

lamindb/_transform.py

@@ -1,11 +1,17 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import Run, Transform
-from lnschema_core.types import TransformType
 
+from ._parents import _view_parents
 from ._run import delete_run_artifacts
 from .core.versioning import process_is_new_version_of
 
+if TYPE_CHECKING:
+    from lnschema_core.types import TransformType
+
 
 def __init__(transform: Transform, *args, **kwargs):
     if len(args) == len(transform._meta.concrete_fields):
@@ -18,9 +24,7 @@ def __init__(transform: Transform, *args, **kwargs):
     )
     (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
-    type: TransformType | None = (
-        kwargs.pop("type") if "type" in kwargs else TransformType.pipeline
-    )
+    type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
     reference_type: str | None = (
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
@@ -55,19 +59,13 @@ def __init__(transform: Transform, *args, **kwargs):
 
 
 def delete(self) -> None:
-    # set latest_report to None, it's tracked through the latest run
-    latest_report = None
-    if self.latest_report is not None:
-        latest_report = self.latest_report
-        self.latest_report = None
-    source_code = None
-    if self.source_code is not None:
-        source_code = self.source_code
-        self.source_code = None
-    if latest_report is not None or source_code is not None:
+    _source_code_artifact = None
+    if self._source_code_artifact is not None:
+        _source_code_artifact = self._source_code_artifact
+        self._source_code_artifact = None
         self.save()
-    if source_code is not None:
-        source_code.delete(permanent=True)
+    if _source_code_artifact is not None:
+        _source_code_artifact.delete(permanent=True)
     # query all runs and delete their artifacts
     runs = Run.filter(transform=self)
     for run in runs:
@@ -78,10 +76,23 @@ def delete(self) -> None:
 
 
 @property  # type: ignore
+@doc_args(Transform.latest_run.__doc__)
 def latest_run(self) -> Run:
+    """{}"""  # noqa: D415
     return self.runs.order_by("-started_at").first()
 
 
+def view_lineage(self, with_successors: bool = False, distance: int = 5):
+    return _view_parents(
+        record=self,
+        field="name",
+        with_children=with_successors,
+        distance=distance,
+        attr_name="predecessors",
+    )
+
+
 Transform.__init__ = __init__
 Transform.delete = delete
 Transform.latest_run = latest_run
+Transform.view_lineage = view_lineage

lamindb/_view.py

@@ -41,15 +41,17 @@ def view(
         schema_module = importlib.import_module(get_schema_module_name(schema_name))
 
         all_registries = {
-            orm
-            for orm in schema_module.__dict__.values()
-            if inspect.isclass(orm)
-            and issubclass(orm, Record)
-            and orm.__name__ != "Record"
+            registry
+            for registry in schema_module.__dict__.values()
+            if inspect.isclass(registry)
+            and issubclass(registry, Record)
+            and registry is not Record
         }
         if registries is not None:
             filtered_registries = {
-                orm for orm in all_registries if orm.__name__ in registries
+                registry
+                for registry in all_registries
+                if registry.__name__ in registries
             }
         else:
             filtered_registries = all_registries
@@ -59,12 +61,12 @@ def view(
             logger.print("*" * len(section_no_color))
             logger.print(section)
             logger.print("*" * len(section_no_color))
-        for orm in sorted(filtered_registries, key=lambda x: x.__name__):
-            if hasattr(orm, "updated_at"):
-                df = orm.filter().order_by("-updated_at")[:n].df()
+        for registry in sorted(filtered_registries, key=lambda x: x.__name__):
+            if hasattr(registry, "updated_at"):
+                df = registry.filter().order_by("-updated_at")[:n].df()
             else:
                 # need to adjust in the future
-                df = orm.df().iloc[-n:]
+                df = registry.df().iloc[-n:]
             if df.shape[0] > 0:
-                logger.print(colors.blue(colors.bold(orm.__name__)))
+                logger.print(colors.blue(colors.bold(registry.__name__)))
                 show(df)

lamindb/core/__init__.py

@@ -6,6 +6,7 @@ Registries:
    :toctree: .
 
    Record
+   Registry
    QuerySet
    QueryManager
    RecordsList
@@ -66,6 +67,7 @@ from lnschema_core.models import (
     IsVersioned,
     ParamValue,
     Record,
+    Registry,
     TracksRun,
     TracksUpdates,
 )

lamindb/core/_data.py

@@ -14,13 +14,13 @@ from lnschema_core.models import (
     Record,
     Run,
     ULabel,
-    __repr__,
     format_field_value,
+    record_repr,
 )
 
 from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
-from lamindb._record import get_default_str_field
+from lamindb._record import get_name_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
@@ -108,7 +108,7 @@ def describe(self: HasFeatures, print_types: bool = False):
     # )
 
     model_name = self.__class__.__name__
-    msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}\n"
+    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
     prov_msg = ""
 
     fields = self._meta.fields
@@ -129,31 +129,29 @@ def describe(self: HasFeatures, print_types: bool = False):
         # prefetch m-2-m relationships
         self = (
             self.__class__.objects.using(self._state.db)
-            .prefetch_related("feature_sets", "input_of")
+            .prefetch_related("feature_sets", "input_of_runs")
             .get(id=self.id)
         )
 
     # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
         fields_values = [(field, getattr(self, field)) for field in foreign_key_fields]
-        type_str = (
-            lambda attr: f": {attr.__class__.__get_name_with_schema__()}"
-            if print_types
-            else ""
+        type_str = lambda attr: (
+            f": {attr.__class__.__get_name_with_schema__()}" if print_types else ""
         )
         related_msg = "".join(
             [
-                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_default_str_field(attr)))}\n"
+                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_name_field(attr)))}\n"
                 for (field_name, attr) in fields_values
                 if attr is not None
             ]
         )
         prov_msg += related_msg
     # input of
-    if self.id is not None and self.input_of.exists():
-        values = [format_field_value(i.started_at) for i in self.input_of.all()]
+    if self.id is not None and self.input_of_runs.exists():
+        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
         type_str = ": Run" if print_types else ""  # type: ignore
-        prov_msg += f"    .input_of{type_str} = {values}\n"
+        prov_msg += f"    .input_of_runs{type_str} = {values}\n"
     if prov_msg:
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
@@ -210,11 +208,11 @@ def get_labels(
             ).all()
     if flat_names:
         # returns a flat list of names
-        from lamindb._record import get_default_str_field
+        from lamindb._record import get_name_field
 
         values = []
         for v in qs_by_registry.values():
-            values += v.list(get_default_str_field(v))
+            values += v.list(get_name_field(v))
         return values
     if len(registries_to_check) == 1 and registry in qs_by_registry:
         return qs_by_registry[registry]
@@ -253,8 +251,8 @@ def add_labels(
         if feature.dtype.startswith("cat["):
             orm_dict = dict_schema_name_to_model_name(Artifact)
             for reg in feature.dtype.replace("cat[", "").rstrip("]").split("|"):
-                orm = orm_dict.get(reg)
-                records_validated += orm.from_values(records, field=field)
+                registry = orm_dict.get(reg)
+                records_validated += registry.from_values(records, field=field)
 
         # feature doesn't have registries and therefore can't create records from values
         # ask users to pass records
@@ -304,12 +302,12 @@ def add_labels(
             if len(linked_labels) > 0:
                 labels_accessor.remove(*linked_labels)
             labels_accessor.add(*records, through_defaults={"feature_id": feature.id})
-        feature_set_links = get_feature_set_links(self)
-        feature_set_ids = [link.featureset_id for link in feature_set_links.all()]
+        links_feature_set = get_feature_set_links(self)
+        feature_set_ids = [link.featureset_id for link in links_feature_set.all()]
         # get all linked features of type Feature
         feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
         {
-            feature_set_links.filter(featureset_id=feature_set.id)
+            links_feature_set.filter(featureset_id=feature_set.id)
             .one()
             .slot: feature_set.features.all()
             for feature_set in feature_sets
@@ -415,7 +413,7 @@ def _track_run_input(
         # generalize below for more than one data batch
         if len(input_data) == 1:
             if input_data[0].transform is not None:
-                run.transform.parents.add(input_data[0].transform)
+                run.transform.predecessors.add(input_data[0].transform)
 
 
 HasFeatures.describe = describe