lamindb 0.76.5__py3-none-any.whl → 0.76.7__py3-none-any.whl

lamindb/core/__init__.py

@@ -54,6 +54,7 @@ Modules:
 .. autosummary::
    :toctree: .
 
+   loaders
    datasets
    storage
    types
@@ -66,8 +67,6 @@ from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
     FeatureValue,
-    HasFeatures,
-    HasParams,
     HasParents,
     IsVersioned,
     ParamValue,
@@ -89,7 +88,7 @@ from lamindb._query_set import QuerySet, RecordsList
 from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
-from . import _data, datasets, exceptions, fields, subsettings, types
+from . import _data, datasets, exceptions, fields, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_context.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import builtins
 import hashlib
-import os
 from datetime import datetime, timezone
 from pathlib import Path, PurePath
 from typing import TYPE_CHECKING
@@ -12,14 +11,15 @@ from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 from lnschema_core import Run, Transform, ids
 from lnschema_core.ids import base62_12
+from lnschema_core.models import format_field_value
 from lnschema_core.users import current_user_id
 
 from ._settings import settings
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
 from .exceptions import (
-    MissingContext,
-    NotebookFileNotSavedToDisk,
+    MissingContextUID,
+    NotebookNotSaved,
     NotebookNotSavedError,
     NoTitleError,
     TrackNotCalled,
@@ -81,21 +81,30 @@ def get_notebook_name_colab() -> str:
     return name.rstrip(".ipynb")
 
 
-def raise_missing_context(transform_type: str, key: str) -> None:
+def raise_missing_context(transform_type: str, key: str) -> bool:
     transform = Transform.filter(key=key).latest_version().first()
     if transform is None:
         new_uid = f"{base62_12()}0000"
-        message = f"To track this {transform_type}, set\n\n"
+        message = f"To track this {transform_type}, copy & paste the below into the current cell and re-run it\n\n"
+        message += f'ln.context.uid = "{new_uid}"\nln.context.track()'
     else:
         uid = transform.uid
         suid, vuid = uid[: Transform._len_stem_uid], uid[Transform._len_stem_uid :]
         new_vuid = increment_base62(vuid)
         new_uid = f"{suid}{new_vuid}"
-        message = f"You already have a {transform_type} version family with key '{key}', suid '{transform.stem_uid}' & name '{transform.name}'.\n\n- to create a new {transform_type} version family, rename your file and rerun: ln.context.track()\n- to bump the version, set: "
-    message += f'ln.context.uid = "{new_uid}"'
+        message = f"You already have a version family with key '{key}' (stem_uid='{transform.stem_uid}').\n\n- to make a revision, set `ln.context.uid = '{new_uid}'`\n- to start a new version family, rename your file and rerun: `ln.context.track()`"
     if transform_type == "notebook":
-        message += "\n\nRestart your notebook if you want consecutive cell execution."
-    raise MissingContext(message)
+        print(f"→ {message}\n")
+        response = input("→ Ready to re-run? (y/n)")
+        if response == "y":
+            logger.important(
+                "Note: Restart your notebook if you want consecutive cell execution"
+            )
+            return True
+        raise MissingContextUID("Please follow the instructions.")
+    else:
+        raise MissingContextUID(message)
+    return False
 
 
 def pretty_pypackages(dependencies: dict) -> str:
@@ -242,7 +251,7 @@ class Context:
                     ):
                         better_version = bump_version_function(self.version)
                         raise SystemExit(
-                            f"Version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
+                            f"Version '{self.version}' is already taken by Transform(uid='{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
                         )
             elif transform_settings_are_set:
                 stem_uid, self.version = (
@@ -280,7 +289,9 @@ class Context:
                 # if no error is raised, the transform is tracked
                 is_tracked = True
             if not is_tracked:
-                raise_missing_context(transform_type, key)
+                early_return = raise_missing_context(transform_type, key)
+                if early_return:
+                    return None
         else:
             if transform.type in {"notebook", "script"}:
                 raise ValueError(
@@ -293,10 +304,10 @@ class Context:
                 transform_exists = Transform.filter(id=transform.id).first()
             if transform_exists is None:
                 transform.save()
-                self._logging_message += f"created Transform('{transform.uid}')"
+                self._logging_message += f"created Transform(uid='{transform.uid}')"
                 transform_exists = transform
             else:
-                self._logging_message += f"loaded Transform('{transform.uid}')"
+                self._logging_message += f"loaded Transform(uid='{transform.uid}')"
             self._transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
@@ -311,7 +322,9 @@ class Context:
             )
             if run is not None:  # loaded latest run
                 run.started_at = datetime.now(timezone.utc)  # update run time
-                self._logging_message += f" & loaded Run('{run.started_at}')"
+                self._logging_message += (
+                    f" & loaded Run(started_at={format_field_value(run.started_at)})"
+                )
 
         if run is None:  # create new run
             run = Run(
@@ -319,7 +332,9 @@ class Context:
                 params=params,
             )
             run.started_at = datetime.now(timezone.utc)
-            self._logging_message += f" & created Run('{run.started_at}')"
+            self._logging_message += (
+                f" & created Run(started_at={format_field_value(run.started_at)})"
+            )
         # can only determine at ln.finish() if run was consecutive in
         # interactive session, otherwise, is consecutive
         run.is_consecutive = True if is_run_from_ipython else None
@@ -432,7 +447,7 @@ class Context:
                 reference_type=transform_ref_type,
                 type=transform_type,
             ).save()
-            self._logging_message += f"created Transform('{transform.uid}')"
+            self._logging_message += f"created Transform(uid='{transform.uid}')"
         else:
             uid = transform.uid
             # check whether the transform file has been renamed
@@ -473,7 +488,9 @@ class Context:
                     if condition:
                         bump_revision = True
                     else:
-                        self._logging_message += f"loaded Transform('{transform.uid}')"
+                        self._logging_message += (
+                            f"loaded Transform(uid='{transform.uid}')"
+                        )
                 if bump_revision:
                     change_type = (
                         "Re-running saved notebook"
@@ -490,7 +507,7 @@ class Context:
                         f'ln.context.uid = "{suid}{new_vuid}"'
                     )
             else:
-                self._logging_message += f"loaded Transform('{transform.uid}')"
+                self._logging_message += f"loaded Transform(uid='{transform.uid}')"
         self._transform = transform
 
     def finish(self, ignore_non_consecutive: None | bool = None) -> None:
@@ -502,7 +519,7 @@ class Context:
         When called in the last cell of a notebook:
 
         - prompts for user input if not consecutively executed
-        - requires to save the notebook in your editor
+        - requires to save the notebook in your editor right before
         - saves a run report: `run.report`
 
         Args:
@@ -524,6 +541,11 @@ class Context:
         def get_seconds_since_modified(filepath) -> float:
             return datetime.now().timestamp() - filepath.stat().st_mtime
 
+        def get_shortcut() -> str:
+            import platform
+
+            return "CMD + s" if platform.system() == "Darwin" else "CTRL + s"
+
         if context.run is None:
             raise TrackNotCalled("Please run `ln.context.track()` before `ln.finish()`")
         if context._path is None:
@@ -537,8 +559,8 @@ class Context:
             return None
         if is_run_from_ipython:  # notebooks
             if get_seconds_since_modified(context._path) > 2 and not ln_setup._TESTING:
-                raise NotebookFileNotSavedToDisk(
-                    "Please save the notebook manually in your editor right before running `ln.context.finish()`"
+                raise NotebookNotSaved(
+                    f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) right before calling `ln.context.finish()`"
                 )
         save_context_core(
             run=context.run,

lamindb/core/_data.py

@@ -10,7 +10,6 @@ from lnschema_core.models import (
     Collection,
     Feature,
     FeatureSet,
-    HasFeatures,
     Record,
     Run,
     ULabel,
@@ -40,10 +39,13 @@ from .schema import (
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
 
+
 WARNING_RUN_TRANSFORM = (
-    "no run & transform get linked, consider calling ln.context.track()"
+    "no run & transform got linked, call `ln.context.track()` & re-run`"
 )
 
+WARNING_NO_INPUT = "run input wasn't tracked, call `ln.context.track()` and re-run"
+
 
 def get_run(run: Run | None) -> Run | None:
     if run is None:
@@ -96,9 +98,14 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
         bulk_create(links, ignore_conflicts=True)
 
 
-@doc_args(HasFeatures.describe.__doc__)
-def describe(self: HasFeatures, print_types: bool = False):
+@doc_args(Artifact.describe.__doc__)
+def describe(self: Artifact, print_types: bool = False):
     """{}"""  # noqa: D415
+    model_name = self.__class__.__name__
+    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
+    if self._state.db is not None and self._state.db != "default":
+        msg += f"  {colors.italic('Database instance')}\n"
+        msg += f"    slug: {self._state.db}\n"
     # prefetch all many-to-many relationships
     # doesn't work for describing using artifact
     # self = (
@@ -109,10 +116,7 @@ def describe(self: HasFeatures, print_types: bool = False):
     #     .get(id=self.id)
     # )
 
-    model_name = self.__class__.__name__
-    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
     prov_msg = ""
-
     fields = self._meta.fields
     direct_fields = []
     foreign_key_fields = []
@@ -129,9 +133,14 @@ def describe(self: HasFeatures, print_types: bool = False):
             .get(id=self.id)
         )
         # prefetch m-2-m relationships
+        many_to_many_fields = []
+        if isinstance(self, (Collection, Artifact)):
+            many_to_many_fields.append("input_of_runs")
+        if isinstance(self, Artifact):
+            many_to_many_fields.append("feature_sets")
         self = (
             self.__class__.objects.using(self._state.db)
-            .prefetch_related("feature_sets", "input_of_runs")
+            .prefetch_related(*many_to_many_fields)
             .get(id=self.id)
         )
 
@@ -149,20 +158,32 @@ def describe(self: HasFeatures, print_types: bool = False):
             ]
         )
         prov_msg += related_msg
-    # input of
-    if self.id is not None and self.input_of_runs.exists():
-        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
-        type_str = ": Run" if print_types else ""  # type: ignore
-        prov_msg += f"    .input_of_runs{type_str} = {values}\n"
     if prov_msg:
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
+
+    # input of runs
+    input_of_message = ""
+    if self.id is not None and self.input_of_runs.exists():
+        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
+        type_str = ": Run" if print_types else ""  # type: ignore
+        input_of_message += f"    .input_of_runs{type_str} = {', '.join(values)}\n"
+    if input_of_message:
+        msg += f"  {colors.italic('Usage')}\n"
+        msg += input_of_message
+
+    # labels
     msg += print_labels(self, print_types=print_types)
-    msg += print_features(  # type: ignore
-        self,
-        print_types=print_types,
-        print_params=hasattr(self, "type") and self.type == "model",
-    )
+
+    # features
+    if isinstance(self, Artifact):
+        msg += print_features(  # type: ignore
+            self,
+            print_types=print_types,
+            print_params=hasattr(self, "type") and self.type == "model",
+        )
+
+    # print entire message
     logger.print(msg)
 
 
@@ -328,7 +349,7 @@ def add_labels(
 
 
 def _track_run_input(
-    data: HasFeatures | Iterable[HasFeatures],
+    data: Artifact | Collection | Iterable[Artifact] | Iterable[Collection],
     is_run_input: bool | None = None,
     run: Run | None = None,
 ):
@@ -340,12 +361,14 @@ def _track_run_input(
     elif run is None:
         run = context.run
     # consider that data is an iterable of Data
-    data_iter: Iterable[HasFeatures] = [data] if isinstance(data, HasFeatures) else data
+    data_iter: Iterable[Artifact] | Iterable[Collection] = (
+        [data] if isinstance(data, (Artifact, Collection)) else data
+    )
     track_run_input = False
     input_data = []
     if run is not None:
         # avoid cycles: data can't be both input and output
-        def is_valid_input(data: HasFeatures):
+        def is_valid_input(data: Artifact | Collection):
             return (
                 data.run_id != run.id
                 and not data._state.adding
@@ -364,10 +387,7 @@ def _track_run_input(
         # we don't have a run record
         if run is None:
             if settings.track_run_inputs:
-                logger.hint(
-                    "you can auto-track these data as a run input by calling"
-                    " `ln.context.track()`"
-                )
+                logger.warning(WARNING_NO_INPUT)
         # assume we have a run record
         else:
             # assume there is non-cyclic candidate input data
@@ -416,7 +436,3 @@ def _track_run_input(
         if len(input_data) == 1:
             if input_data[0].transform is not None:
                 run.transform.predecessors.add(input_data[0].transform)
-
-
-HasFeatures.describe = describe
-HasFeatures.view_lineage = view_lineage

lamindb/core/_feature_manager.py

@@ -19,11 +19,7 @@ from lnschema_core.models import (
     Collection,
     Feature,
     FeatureManager,
-    FeatureManagerArtifact,
-    FeatureManagerCollection,
     FeatureValue,
-    HasFeatures,
-    HasParams,
     LinkORM,
     Param,
     ParamManager,
@@ -116,7 +112,7 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     return links_feature_set
 
 
-def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
+def get_link_attr(link: LinkORM | type[LinkORM], data: Artifact | Collection) -> str:
     link_model_name = link.__class__.__name__
     if link_model_name in {"Registry", "ModelBase"}:  # we passed the type of the link
         link_model_name = link.__name__
@@ -137,7 +133,7 @@ def custom_aggregate(field, using: str):
 
 
 def print_features(
-    self: HasFeatures | HasParams,
+    self: Artifact | Collection,
     print_types: bool = False,
     to_dict: bool = False,
     print_params: bool = False,
@@ -362,7 +358,7 @@ def __getitem__(self, slot) -> QuerySet:
 
 
 def filter_base(cls, **expression):
-    if cls in {FeatureManagerArtifact, FeatureManagerCollection}:
+    if cls is FeatureManager:
         model = Feature
         value_model = FeatureValue
     else:
@@ -394,10 +390,11 @@ def filter_base(cls, **expression):
                 new_expression["ulabels"] = label
             else:
                 raise NotImplementedError
-    if cls == FeatureManagerArtifact or cls == ParamManagerArtifact:
+    if cls == FeatureManager or cls == ParamManagerArtifact:
         return Artifact.filter(**new_expression)
-    elif cls == FeatureManagerCollection:
-        return Collection.filter(**new_expression)
+    # might renable something similar in the future
+    # elif cls == FeatureManagerCollection:
+    #     return Collection.filter(**new_expression)
     elif cls == ParamManagerRun:
         return Run.filter(**new_expression)
 
@@ -791,9 +788,11 @@ def _add_set_from_mudata(
     self._host.save()
 
 
-def _add_from(self, data: HasFeatures):
+def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
     """Transfer features from a artifact or collection."""
-    # This only covers feature sets, though.
+    # This only covers feature sets
+    if transfer_logs is None:
+        transfer_logs = {"mapped": [], "transferred": []}
     using_key = settings._using_key
     for slot, feature_set in data.features._feature_set_by_slot.items():
         members = feature_set.members
@@ -809,21 +808,25 @@ def _add_from(self, data: HasFeatures):
         # create records from ontology_id
         if hasattr(registry, "_ontology_id_field") and len(member_uids) > 0:
             # create from bionty
-            save(registry.from_values(member_uids, field=field))
+            members_records = registry.from_values(member_uids, field=field)
+            save([r for r in members_records if r._state.adding])
         validated = registry.validate(member_uids, field=field, mute=True)
         new_members_uids = list(compress(member_uids, ~validated))
         new_members = members.filter(**{f"{field}__in": new_members_uids}).all()
         n_new_members = len(new_members)
         if n_new_members > 0:
-            mute = True if n_new_members > 10 else False
             # transfer foreign keys needs to be run before transfer to default db
-            transfer_fk_to_default_db_bulk(new_members, using_key)
+            transfer_fk_to_default_db_bulk(
+                new_members, using_key, transfer_logs=transfer_logs
+            )
             for feature in new_members:
                 # not calling save=True here as in labels, because want to
                 # bulk save below
                 # transfer_fk is set to False because they are already transferred
                 # in the previous step transfer_fk_to_default_db_bulk
-                transfer_to_default_db(feature, using_key, mute=mute, transfer_fk=False)
+                transfer_to_default_db(
+                    feature, using_key, transfer_fk=False, transfer_logs=transfer_logs
+                )
             logger.info(f"saving {n_new_members} new {registry.__name__} records")
             save(new_members)
 

lamindb/core/_label_manager.py

@@ -20,12 +20,12 @@ from ._settings import settings
 from .schema import dict_related_model_to_related_name
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Artifact, Collection, HasFeatures, Record
+    from lnschema_core.models import Artifact, Collection, Record
 
     from lamindb._query_set import QuerySet
 
 
-def get_labels_as_dict(self: HasFeatures, links: bool = False):
+def get_labels_as_dict(self: Artifact | Collection, links: bool = False):
     exclude_set = {
         "feature_sets",
         "artifacts",
@@ -57,7 +57,9 @@ def get_labels_as_dict(self: HasFeatures, links: bool = False):
     return labels
 
 
-def print_labels(self: HasFeatures, field: str = "name", print_types: bool = False):
+def print_labels(
+    self: Artifact | Collection, field: str = "name", print_types: bool = False
+):
     labels_msg = ""
     for related_name, (related_model, labels) in get_labels_as_dict(self).items():
         # there is a try except block here to deal with schema inconsistencies
@@ -96,7 +98,8 @@ def validate_labels(labels: QuerySet | list | dict):
         # save labels from ontology_ids
         if hasattr(registry, "_ontology_id_field") and len(label_uids) > 0:
             try:
-                save(registry.from_values(label_uids, field=field))
+                labels_records = registry.from_values(label_uids, field=field)
+                save([r for r in labels_records if r._state.adding])
             except Exception:  # noqa S110
                 pass
             field = "uid"
@@ -167,22 +170,22 @@ class LabelManager:
 
         return get_labels(self._host, feature=feature, mute=mute, flat_names=flat_names)
 
-    def add_from(self, data: HasFeatures) -> None:
+    def add_from(self, data: Artifact | Collection, transfer_logs: dict = None) -> None:
         """Add labels from an artifact or collection to another artifact or collection.
 
         Examples:
-            >>> file1 = ln.Artifact(pd.DataFrame(index=[0, 1]))
-            >>> file1.save()
-            >>> file2 = ln.Artifact(pd.DataFrame(index=[2, 3]))
-            >>> file2.save()
+            >>> artifact1 = ln.Artifact(pd.DataFrame(index=[0, 1])).save()
+            >>> artifact2 = ln.Artifact(pd.DataFrame(index=[2, 3])).save()
             >>> ulabels = ln.ULabel.from_values(["Label1", "Label2"], field="name")
             >>> ln.save(ulabels)
             >>> labels = ln.ULabel.filter(name__icontains = "label").all()
-            >>> file1.ulabels.set(labels)
-            >>> file2.labels.add_from(file1)
+            >>> artifact1.ulabels.set(labels)
+            >>> artifact2.labels.add_from(artifact1)
         """
         from django.db.utils import ProgrammingError
 
+        if transfer_logs is None:
+            transfer_logs = {"mapped": [], "transferred": []}
         using_key = settings._using_key
         for related_name, (_, labels) in get_labels_as_dict(data).items():
             labels = labels.all()
@@ -195,7 +198,9 @@ class LabelManager:
                 features = set()
                 _, new_labels = validate_labels(labels)
                 if len(new_labels) > 0:
-                    transfer_fk_to_default_db_bulk(new_labels, using_key)
+                    transfer_fk_to_default_db_bulk(
+                        new_labels, using_key, transfer_logs=transfer_logs
+                    )
                 for label in labels:
                     # if the link table doesn't follow this convention, we'll ignore it
                     if not hasattr(label, f"links_{data_name_lower}"):
@@ -212,7 +217,7 @@ class LabelManager:
                     label_returned = transfer_to_default_db(
                         label,
                         using_key,
-                        mute=True,
+                        transfer_logs=transfer_logs,
                         transfer_fk=False,
                         save=True,
                     )
@@ -223,10 +228,15 @@ class LabelManager:
                 # treat features
                 _, new_features = validate_labels(list(features))
                 if len(new_features) > 0:
-                    transfer_fk_to_default_db_bulk(new_features, using_key)
+                    transfer_fk_to_default_db_bulk(
+                        new_features, using_key, transfer_logs=transfer_logs
+                    )
                     for feature in new_features:
                         transfer_to_default_db(
-                            feature, using_key, mute=True, transfer_fk=False
+                            feature,
+                            using_key,
+                            transfer_logs=transfer_logs,
+                            transfer_fk=False,
                         )
                     save(new_features)
                 if hasattr(self._host, related_name):

lamindb/core/_settings.py

@@ -52,7 +52,7 @@ class Settings:
         return creation_settings
 
     track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.cache()` and `.backed()`.
+    """Track files as input upon `.load()`, `.cache()` and `.open()`.
 
     Requires a global run context with :func:`~lamindb.core.Context.track` was created!
 

lamindb/core/exceptions.py

@@ -9,7 +9,7 @@ The registry base class:
    ValidationError
    NotebookNotSavedError
    NoTitleError
-   MissingContext
+   MissingContextUID
    UpdateContext
    IntegrityError
 
@@ -20,7 +20,7 @@ class TrackNotCalled(SystemExit):
     pass
 
 
-class NotebookFileNotSavedToDisk(SystemExit):
+class NotebookNotSaved(SystemExit):
     pass
 
 
@@ -65,7 +65,7 @@ class NoTitleError(Exception):
     pass
 
 
-class MissingContext(SystemExit):
+class MissingContextUID(SystemExit):
     """User didn't define transform settings."""
 
     pass