lamindb 0.76.4__py3-none-any.whl → 0.76.6__py3-none-any.whl

lamindb/_transform.py

@@ -37,8 +37,25 @@ def __init__(transform: Transform, *args, **kwargs):
             "Only name, key, version, type, revises, reference, "
             f"reference_type can be passed, but you passed: {kwargs}"
         )
-    if revises is None and key is not None:
-        revises = Transform.filter(key=key).order_by("-created_at").first()
+    if revises is None:
+        if key is not None:
+            revises = (
+                Transform.filter(key=key, is_latest=True)
+                .order_by("-created_at")
+                .first()
+            )
+        elif uid is not None:
+            revises = (
+                Transform.filter(uid__startswith=uid[:-4], is_latest=True)
+                .order_by("-created_at")
+                .first()
+            )
+    if revises is not None and uid is not None and uid == revises.uid:
+        from ._record import init_self_from_db, update_attributes
+
+        init_self_from_db(transform, revises)
+        update_attributes(transform, {"name": name})
+        return None
     if revises is not None and key is not None and revises.key != key:
         note = message_update_key_in_version_family(
             suid=revises.stem_uid,

lamindb/core/__init__.py

@@ -66,8 +66,6 @@ from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
     FeatureValue,
-    HasFeatures,
-    HasParams,
     HasParents,
     IsVersioned,
     ParamValue,

lamindb/core/_context.py

@@ -2,15 +2,16 @@ from __future__ import annotations
 
 import builtins
 import hashlib
-import os
 from datetime import datetime, timezone
 from pathlib import Path, PurePath
 from typing import TYPE_CHECKING
 
+import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 from lnschema_core import Run, Transform, ids
 from lnschema_core.ids import base62_12
+from lnschema_core.models import format_field_value
 from lnschema_core.users import current_user_id
 
 from ._settings import settings
@@ -111,7 +112,18 @@ def pretty_pypackages(dependencies: dict) -> str:
 class Context:
     """Run context.
 
-    Bundles all metadata to track run contexts.
+    Enables convenient data lineage tracking by managing a transform & run
+    upon :meth:`~lamindb.core.Context.track` & :meth:`~lamindb.core.Context.finish`.
+
+    Examples:
+
+        Is typically used via :class:`~lamindb.context`:
+
+        >>> import lamindb as ln
+        >>> ln.context.track()
+        >>> # do things while tracking data lineage
+        >>> ln.context.finish()
+
     """
 
     def __init__(self):
@@ -165,42 +177,35 @@ class Context:
         self,
         *,
         params: dict | None = None,
-        transform: Transform | None = None,
         new_run: bool | None = None,
         path: str | None = None,
+        transform: Transform | None = None,
     ) -> None:
-        """Track notebook or script run.
-
-        Creates or loads a global :class:`~lamindb.Run` that enables data
-        lineage tracking.
+        """Starts data lineage tracking for a run.
 
-        Saves source code and compute environment.
+        - sets :attr:`~lamindb.core.Context.transform` &
+          :attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
+          `Run` records
+        - saves compute environment as a `requirements.txt` file: `run.environment`
 
-        If :attr:`~lamindb.core.Settings.sync_git_repo` is set, will first check
-        whether the script exists in the git repository and add a link.
+        If :attr:`~lamindb.core.Settings.sync_git_repo` is set, checks whether a
+        script-like transform exists in a git repository and links it.
 
         Args:
             params: A dictionary of parameters to track for the run.
-            transform: Can be of type `"pipeline"` or `"notebook"`
-                (:class:`~lamindb.core.types.TransformType`).
             new_run: If `False`, loads latest run of transform
                 (default notebook), if `True`, creates new run (default pipeline).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
+            transform: Useful to track an abstract pipeline.
 
         Examples:
 
-            To track a notebook or script, call:
+            To track the run of a notebook or script, call:
 
             >>> import lamindb as ln
             >>> ln.context.track()
 
-            If you'd like to track an abstract pipeline run, pass a
-            :class:`~lamindb.Transform` object of ``type`` ``"pipeline"``:
-
-            >>> ln.Transform(name="Cell Ranger", version="2", type="pipeline").save()
-            >>> transform = ln.Transform.get(name="Cell Ranger", version="2")
-            >>> ln.context.track(transform=transform)
         """
         self._path = None
         if transform is None:
@@ -237,7 +242,7 @@ class Context:
                     ):
                         better_version = bump_version_function(self.version)
                         raise SystemExit(
-                            f"Version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
+                            f"Version '{self.version}' is already taken by Transform(uid='{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
                         )
             elif transform_settings_are_set:
                 stem_uid, self.version = (
@@ -288,10 +293,10 @@ class Context:
                 transform_exists = Transform.filter(id=transform.id).first()
             if transform_exists is None:
                 transform.save()
-                self._logging_message += f"created Transform('{transform.uid}')"
+                self._logging_message += f"created Transform(uid='{transform.uid}')"
                 transform_exists = transform
             else:
-                self._logging_message += f"loaded Transform('{transform.uid}')"
+                self._logging_message += f"loaded Transform(uid='{transform.uid}')"
             self._transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
@@ -306,7 +311,9 @@ class Context:
             )
             if run is not None:  # loaded latest run
                 run.started_at = datetime.now(timezone.utc)  # update run time
-                self._logging_message += f" & loaded Run('{run.started_at}')"
+                self._logging_message += (
+                    f" & loaded Run(started_at={format_field_value(run.started_at)})"
+                )
 
         if run is None:  # create new run
             run = Run(
@@ -314,7 +321,9 @@ class Context:
                 params=params,
             )
             run.started_at = datetime.now(timezone.utc)
-            self._logging_message += f" & created Run('{run.started_at}')"
+            self._logging_message += (
+                f" & created Run(started_at={format_field_value(run.started_at)})"
+            )
         # can only determine at ln.finish() if run was consecutive in
         # interactive session, otherwise, is consecutive
         run.is_consecutive = True if is_run_from_ipython else None
@@ -427,7 +436,7 @@ class Context:
                 reference_type=transform_ref_type,
                 type=transform_type,
             ).save()
-            self._logging_message += f"created Transform('{transform.uid}')"
+            self._logging_message += f"created Transform(uid='{transform.uid}')"
         else:
             uid = transform.uid
             # check whether the transform file has been renamed
@@ -468,7 +477,9 @@ class Context:
                     if condition:
                         bump_revision = True
                     else:
-                        self._logging_message += f"loaded Transform('{transform.uid}')"
+                        self._logging_message += (
+                            f"loaded Transform(uid='{transform.uid}')"
+                        )
                 if bump_revision:
                     change_type = (
                         "Re-running saved notebook"
@@ -485,13 +496,34 @@ class Context:
                         f'ln.context.uid = "{suid}{new_vuid}"'
                     )
             else:
-                self._logging_message += f"loaded Transform('{transform.uid}')"
+                self._logging_message += f"loaded Transform(uid='{transform.uid}')"
         self._transform = transform
 
-    def finish(self) -> None:
-        """Mark a tracked run as finished.
+    def finish(self, ignore_non_consecutive: None | bool = None) -> None:
+        """Mark the run context as finished.
+
+        - writes a timestamp: `run.finished_at`
+        - saves the source code: `transform.source_code`
+
+        When called in the last cell of a notebook:
+
+        - prompts for user input if not consecutively executed
+        - requires to save the notebook in your editor
+        - saves a run report: `run.report`
+
+        Args:
+            ignore_non_consecutive: Whether to ignore if a notebook was non-consecutively executed.
+
+        Examples:
+
+            >>> import lamindb as ln
+            >>> ln.context.track()
+            >>> # do things while tracking data lineage
+            >>> ln.context.finish()
+
+        See Also:
+            `lamin save script.py` or `lamin save notebook.ipynb` → `docs </cli#lamin-save>`__
 
-        Saves source code and, for notebooks, a run report to your default storage location.
         """
         from lamindb._finish import save_context_core
 
@@ -510,18 +542,16 @@ class Context:
             # nothing else to do
             return None
         if is_run_from_ipython:  # notebooks
-            if (
-                get_seconds_since_modified(context._path) > 3
-                and os.getenv("LAMIN_TESTING") is None
-            ):
+            if get_seconds_since_modified(context._path) > 2 and not ln_setup._TESTING:
                 raise NotebookFileNotSavedToDisk(
-                    "Please save the notebook manually in your editor right before running `ln.finish()`"
+                    "Please save the notebook manually in your editor right before running `ln.context.finish()`"
                 )
         save_context_core(
             run=context.run,
             transform=context.run.transform,
             filepath=context._path,
             finished_at=True,
+            ignore_non_consecutive=ignore_non_consecutive,
         )
 
 

lamindb/core/_data.py

@@ -10,7 +10,6 @@ from lnschema_core.models import (
     Collection,
     Feature,
     FeatureSet,
-    HasFeatures,
     Record,
     Run,
     ULabel,
@@ -96,9 +95,14 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
         bulk_create(links, ignore_conflicts=True)
 
 
-@doc_args(HasFeatures.describe.__doc__)
-def describe(self: HasFeatures, print_types: bool = False):
+@doc_args(Artifact.describe.__doc__)
+def describe(self: Artifact, print_types: bool = False):
     """{}"""  # noqa: D415
+    model_name = self.__class__.__name__
+    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
+    if self._state.db is not None and self._state.db != "default":
+        msg += f"  {colors.italic('Database instance')}\n"
+        msg += f"    slug: {self._state.db}\n"
     # prefetch all many-to-many relationships
     # doesn't work for describing using artifact
     # self = (
@@ -109,10 +113,7 @@ def describe(self: HasFeatures, print_types: bool = False):
     #     .get(id=self.id)
     # )
 
-    model_name = self.__class__.__name__
-    msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
     prov_msg = ""
-
     fields = self._meta.fields
     direct_fields = []
     foreign_key_fields = []
@@ -129,9 +130,14 @@ def describe(self: HasFeatures, print_types: bool = False):
             .get(id=self.id)
         )
         # prefetch m-2-m relationships
+        many_to_many_fields = []
+        if isinstance(self, (Collection, Artifact)):
+            many_to_many_fields.append("input_of_runs")
+        if isinstance(self, Artifact):
+            many_to_many_fields.append("feature_sets")
         self = (
             self.__class__.objects.using(self._state.db)
-            .prefetch_related("feature_sets", "input_of_runs")
+            .prefetch_related(*many_to_many_fields)
             .get(id=self.id)
         )
 
@@ -149,20 +155,32 @@ def describe(self: HasFeatures, print_types: bool = False):
             ]
         )
         prov_msg += related_msg
-    # input of
-    if self.id is not None and self.input_of_runs.exists():
-        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
-        type_str = ": Run" if print_types else ""  # type: ignore
-        prov_msg += f"    .input_of_runs{type_str} = {values}\n"
     if prov_msg:
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
+
+    # input of runs
+    input_of_message = ""
+    if self.id is not None and self.input_of_runs.exists():
+        values = [format_field_value(i.started_at) for i in self.input_of_runs.all()]
+        type_str = ": Run" if print_types else ""  # type: ignore
+        input_of_message += f"    .input_of_runs{type_str} = {', '.join(values)}\n"
+    if input_of_message:
+        msg += f"  {colors.italic('Usage')}\n"
+        msg += input_of_message
+
+    # labels
     msg += print_labels(self, print_types=print_types)
-    msg += print_features(  # type: ignore
-        self,
-        print_types=print_types,
-        print_params=hasattr(self, "type") and self.type == "model",
-    )
+
+    # features
+    if isinstance(self, Artifact):
+        msg += print_features(  # type: ignore
+            self,
+            print_types=print_types,
+            print_params=hasattr(self, "type") and self.type == "model",
+        )
+
+    # print entire message
     logger.print(msg)
 
 
@@ -328,7 +346,7 @@ def add_labels(
 
 
 def _track_run_input(
-    data: HasFeatures | Iterable[HasFeatures],
+    data: Artifact | Collection | Iterable[Artifact] | Iterable[Collection],
     is_run_input: bool | None = None,
     run: Run | None = None,
 ):
@@ -340,12 +358,14 @@ def _track_run_input(
     elif run is None:
         run = context.run
     # consider that data is an iterable of Data
-    data_iter: Iterable[HasFeatures] = [data] if isinstance(data, HasFeatures) else data
+    data_iter: Iterable[Artifact] | Iterable[Collection] = (
+        [data] if isinstance(data, (Artifact, Collection)) else data
+    )
     track_run_input = False
     input_data = []
     if run is not None:
         # avoid cycles: data can't be both input and output
-        def is_valid_input(data: HasFeatures):
+        def is_valid_input(data: Artifact | Collection):
             return (
                 data.run_id != run.id
                 and not data._state.adding
@@ -416,7 +436,3 @@ def _track_run_input(
         if len(input_data) == 1:
             if input_data[0].transform is not None:
                 run.transform.predecessors.add(input_data[0].transform)
-
-
-HasFeatures.describe = describe
-HasFeatures.view_lineage = view_lineage

lamindb/core/_feature_manager.py

@@ -19,11 +19,7 @@ from lnschema_core.models import (
     Collection,
     Feature,
     FeatureManager,
-    FeatureManagerArtifact,
-    FeatureManagerCollection,
     FeatureValue,
-    HasFeatures,
-    HasParams,
     LinkORM,
     Param,
     ParamManager,
@@ -116,7 +112,7 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     return links_feature_set
 
 
-def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
+def get_link_attr(link: LinkORM | type[LinkORM], data: Artifact | Collection) -> str:
     link_model_name = link.__class__.__name__
     if link_model_name in {"Registry", "ModelBase"}:  # we passed the type of the link
         link_model_name = link.__name__
@@ -137,7 +133,7 @@ def custom_aggregate(field, using: str):
 
 
 def print_features(
-    self: HasFeatures | HasParams,
+    self: Artifact | Collection,
     print_types: bool = False,
     to_dict: bool = False,
     print_params: bool = False,
@@ -362,7 +358,7 @@ def __getitem__(self, slot) -> QuerySet:
 
 
 def filter_base(cls, **expression):
-    if cls in {FeatureManagerArtifact, FeatureManagerCollection}:
+    if cls is FeatureManager:
         model = Feature
         value_model = FeatureValue
     else:
@@ -394,10 +390,11 @@ def filter_base(cls, **expression):
                 new_expression["ulabels"] = label
             else:
                 raise NotImplementedError
-    if cls == FeatureManagerArtifact or cls == ParamManagerArtifact:
+    if cls == FeatureManager or cls == ParamManagerArtifact:
         return Artifact.filter(**new_expression)
-    elif cls == FeatureManagerCollection:
-        return Collection.filter(**new_expression)
+    # might renable something similar in the future
+    # elif cls == FeatureManagerCollection:
+    #     return Collection.filter(**new_expression)
     elif cls == ParamManagerRun:
         return Run.filter(**new_expression)
 
@@ -791,9 +788,11 @@ def _add_set_from_mudata(
     self._host.save()
 
 
-def _add_from(self, data: HasFeatures):
+def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
     """Transfer features from a artifact or collection."""
-    # This only covers feature sets, though.
+    # This only covers feature sets
+    if transfer_logs is None:
+        transfer_logs = {"mapped": [], "transferred": []}
     using_key = settings._using_key
     for slot, feature_set in data.features._feature_set_by_slot.items():
         members = feature_set.members
@@ -815,15 +814,18 @@ def _add_from(self, data: HasFeatures):
         new_members = members.filter(**{f"{field}__in": new_members_uids}).all()
         n_new_members = len(new_members)
         if n_new_members > 0:
-            mute = True if n_new_members > 10 else False
             # transfer foreign keys needs to be run before transfer to default db
-            transfer_fk_to_default_db_bulk(new_members, using_key)
+            transfer_fk_to_default_db_bulk(
+                new_members, using_key, transfer_logs=transfer_logs
+            )
             for feature in new_members:
                 # not calling save=True here as in labels, because want to
                 # bulk save below
                 # transfer_fk is set to False because they are already transferred
                 # in the previous step transfer_fk_to_default_db_bulk
-                transfer_to_default_db(feature, using_key, mute=mute, transfer_fk=False)
+                transfer_to_default_db(
+                    feature, using_key, transfer_fk=False, transfer_logs=transfer_logs
+                )
             logger.info(f"saving {n_new_members} new {registry.__name__} records")
             save(new_members)
 

lamindb/core/_label_manager.py

@@ -20,12 +20,12 @@ from ._settings import settings
 from .schema import dict_related_model_to_related_name
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Artifact, Collection, HasFeatures, Record
+    from lnschema_core.models import Artifact, Collection, Record
 
     from lamindb._query_set import QuerySet
 
 
-def get_labels_as_dict(self: HasFeatures, links: bool = False):
+def get_labels_as_dict(self: Artifact | Collection, links: bool = False):
     exclude_set = {
         "feature_sets",
         "artifacts",
@@ -57,7 +57,9 @@ def get_labels_as_dict(self: HasFeatures, links: bool = False):
     return labels
 
 
-def print_labels(self: HasFeatures, field: str = "name", print_types: bool = False):
+def print_labels(
+    self: Artifact | Collection, field: str = "name", print_types: bool = False
+):
     labels_msg = ""
     for related_name, (related_model, labels) in get_labels_as_dict(self).items():
         # there is a try except block here to deal with schema inconsistencies
@@ -167,22 +169,22 @@ class LabelManager:
 
         return get_labels(self._host, feature=feature, mute=mute, flat_names=flat_names)
 
-    def add_from(self, data: HasFeatures) -> None:
+    def add_from(self, data: Artifact | Collection, transfer_logs: dict = None) -> None:
         """Add labels from an artifact or collection to another artifact or collection.
 
         Examples:
-            >>> file1 = ln.Artifact(pd.DataFrame(index=[0, 1]))
-            >>> file1.save()
-            >>> file2 = ln.Artifact(pd.DataFrame(index=[2, 3]))
-            >>> file2.save()
+            >>> artifact1 = ln.Artifact(pd.DataFrame(index=[0, 1])).save()
+            >>> artifact2 = ln.Artifact(pd.DataFrame(index=[2, 3])).save()
             >>> ulabels = ln.ULabel.from_values(["Label1", "Label2"], field="name")
             >>> ln.save(ulabels)
             >>> labels = ln.ULabel.filter(name__icontains = "label").all()
-            >>> file1.ulabels.set(labels)
-            >>> file2.labels.add_from(file1)
+            >>> artifact1.ulabels.set(labels)
+            >>> artifact2.labels.add_from(artifact1)
         """
         from django.db.utils import ProgrammingError
 
+        if transfer_logs is None:
+            transfer_logs = {"mapped": [], "transferred": []}
         using_key = settings._using_key
         for related_name, (_, labels) in get_labels_as_dict(data).items():
             labels = labels.all()
@@ -195,7 +197,9 @@ class LabelManager:
                 features = set()
                 _, new_labels = validate_labels(labels)
                 if len(new_labels) > 0:
-                    transfer_fk_to_default_db_bulk(new_labels, using_key)
+                    transfer_fk_to_default_db_bulk(
+                        new_labels, using_key, transfer_logs=transfer_logs
+                    )
                 for label in labels:
                     # if the link table doesn't follow this convention, we'll ignore it
                     if not hasattr(label, f"links_{data_name_lower}"):
@@ -212,7 +216,7 @@ class LabelManager:
                     label_returned = transfer_to_default_db(
                         label,
                         using_key,
-                        mute=True,
+                        transfer_logs=transfer_logs,
                         transfer_fk=False,
                         save=True,
                     )
@@ -223,10 +227,15 @@ class LabelManager:
                 # treat features
                 _, new_features = validate_labels(list(features))
                 if len(new_features) > 0:
-                    transfer_fk_to_default_db_bulk(new_features, using_key)
+                    transfer_fk_to_default_db_bulk(
+                        new_features, using_key, transfer_logs=transfer_logs
+                    )
                     for feature in new_features:
                         transfer_to_default_db(
-                            feature, using_key, mute=True, transfer_fk=False
+                            feature,
+                            using_key,
+                            transfer_logs=transfer_logs,
+                            transfer_fk=False,
                         )
                     save(new_features)
                 if hasattr(self._host, related_name):

lamindb/core/_settings.py

@@ -52,7 +52,7 @@ class Settings:
         return creation_settings
 
     track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.cache()` and `.backed()`.
+    """Track files as input upon `.load()`, `.cache()` and `.open()`.
 
     Requires a global run context with :func:`~lamindb.core.Context.track` was created!
 

lamindb/integrations/_vitessce.py

@@ -73,12 +73,15 @@ def save_vitessce_config(
                 )
             else:
                 dataset_artifacts.append(artifact)
-    # link inputs
+    # the below will be replaced with a `ln.tracked()` decorator soon
     with logger.mute():
-        transform = Transform(name="save_vitessce_config", type="function", version="2")
-        transform.save()
-    run = Run(transform=transform)
-    run.save()
+        transform = Transform(
+            uid="kup03MJBsIVa0001",
+            name="save_vitessce_config",
+            type="function",
+            version="2",
+        ).save()
+    run = Run(transform=transform).save()
     if len(dataset_artifacts) > 1:
         # if we have more datasets, we should create a collection
         # and attach an action to the collection

{lamindb-0.76.4.dist-info → lamindb-0.76.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.4
+Version: 0.76.6
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.73.5
-Requires-Dist: lamindb_setup==0.76.8
+Requires-Dist: lnschema_core==0.74.1
+Requires-Dist: lamindb_setup==0.77.1
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.17.0
+Requires-Dist: lamin_cli==0.17.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -24,7 +24,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.49.1 ; extra == "bionty"
+Requires-Dist: bionty==0.50.1 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"