lamindb 1.4.0__py3-none-any.whl → 1.5.1__py3-none-any.whl

lamindb/core/_context.py

@@ -54,19 +54,21 @@ def get_uid_ext(version: str) -> str:
     return encodebytes(hashlib.md5(version.encode()).digest())[:4]  # noqa: S324
 
 
-def get_notebook_path() -> Path:
+def get_notebook_path() -> tuple[Path, str]:
     from nbproject.dev._jupyter_communicate import (
         notebook_path as get_notebook_path,
     )
 
     path = None
     try:
-        path = get_notebook_path()
-    except Exception:
-        raise RuntimeError(msg_path_failed) from None
+        path, env = get_notebook_path(return_env=True)
+    except ValueError as ve:
+        raise ve
+    except Exception as error:
+        raise RuntimeError(msg_path_failed) from error
     if path is None:
         raise RuntimeError(msg_path_failed) from None
-    return Path(path)
+    return Path(path), env
 
 
 # from https://stackoverflow.com/questions/61901628
@@ -178,20 +180,7 @@ class LogStreamTracker:
 class Context:
     """Run context.
 
-    Enables convenient data lineage tracking by managing a transform & run
-    upon :meth:`~lamindb.core.Context.track` & :meth:`~lamindb.core.Context.finish`.
-
-    Guide: :doc:`/track`
-
-    Examples:
-
-        Is typically used via the global :class:`~lamindb.context` object via `ln.track()` and `ln.finish()`:
-
-        >>> import lamindb as ln
-        >>> ln.track()
-        >>> # do things
-        >>> ln.finish()
-
+    Is the book keeper for :meth:`~lamindb.core.Context.track`.
     """
 
     def __init__(self):
@@ -201,13 +190,14 @@ class Context:
         self._transform: Transform | None = None
         self._run: Run | None = None
         self._path: Path | None = None
-        """A local path to the script that's running."""
+        """A local path to the script or notebook that's running."""
         self._project: Project | None = None
         self._space: Space | None = None
         self._logging_message_track: str = ""
         self._logging_message_imports: str = ""
         self._stream_tracker: LogStreamTracker = LogStreamTracker()
         self._is_finish_retry: bool = False
+        self._notebook_runner: str | None = None
 
     @property
     def transform(self) -> Transform | None:
@@ -265,31 +255,29 @@ class Context:
         """Managed run of context."""
         return self._run
 
-    def track(
+    def _track(
         self,
         transform: str | Transform | None = None,
         *,
-        project: str | None = None,
-        space: str | None = None,
+        project: str | Project | None = None,
+        space: str | Space | None = None,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
     ) -> None:
-        """Track a global run in your compute session.
+        """Track a run of your notebook or script.
 
-        - sets :attr:`~lamindb.core.Context.transform` &
-          :attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
-          `Run` records
-        - writes compute environment to prepare populating: `run.environment`
-        - if :attr:`~lamindb.core.Settings.sync_git_repo` is set, checks whether a script-like
-          transform exists in a git repository and links it
+        Populates the global run :class:`~lamindb.context` by managing `Transform` & `Run` records and caching the compute environment.
 
-        Guide: :doc:`/track`
+        If :attr:`~lamindb.core.Settings.sync_git_repo` is set, checks whether a script-like transform exists in a git repository and links it.
 
         Args:
-            transform: A transform `uid` or record. If `None`, manages the `transform` based on the script or notebook that calls `ln.track()`.
-            project: A project `name` or `uid` for labeling entities created during the run.
-            space: A space `name` or `uid` for creating entities during the run. This doesn't affect bionty entities given they should typically be commonly accessible.
+            transform: A transform (stem) `uid` (or record). If `None`, auto-creates a `transform` with its `uid`.
+            project: A project, its `name` or `uid` for labeling entities created during the run.
+            space: A restricted space, its `name` or `uid` for creating sensitive entities are created during the run.
+                The default is the common `"All"` space that every LaminDB instance has.
+                The `space` argument doesn't affect `Storage`, `ULabel`, `Feature`, `Schema`, `Param` and bionty entities as these provide structure that should typically be commonly accessible.
+                If you want to manually move entities to a different space, set the `.space` field (:doc:`docs:access`).
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads the latest run of transform
                 (default notebook), if `True`, creates new run (default non-notebook).
@@ -301,11 +289,15 @@ class Context:
             To track the run of a notebook or script, call::
 
                 ln.track()
+                #> → created Transform('Onv04I53OgtT0000'), started new Run('dpSfd7Ds...') at 2025-04-25 11:00:03 UTC
+                #> • recommendation: to identify the notebook across renames, pass the uid: ln.track("Onv04I53OgtT")
 
-            If you want to ensure a single version history across renames of the notebook or script, pass the auto-generated `uid` that you'll find in the logs::
+            Ensure one version history across file renames::
 
-                ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
+                ln.track("Onv04I53OgtT")
+                #> → created Transform('Onv04I53OgtT0000'), started new Run('dpSfd7Ds...') at 2025-04-25 11:00:03 UTC
 
+            More examples: :doc:`/track`
         """
         from lamindb.models import Project, Space
 
@@ -318,26 +310,41 @@ class Context:
         if project is None:
             project = os.environ.get("LAMIN_CURRENT_PROJECT")
         if project is not None:
-            project_record = Project.filter(
-                Q(name=project) | Q(uid=project)
-            ).one_or_none()
-            if project_record is None:
-                raise InvalidArgument(
-                    f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+            if isinstance(project, Project):
+                assert project._state.adding is False, (  # noqa: S101
+                    "Project must be saved before passing it to track()"
                 )
+                project_record = project
+            else:
+                project_record = Project.filter(
+                    Q(name=project) | Q(uid=project)
+                ).one_or_none()
+                if project_record is None:
+                    raise InvalidArgument(
+                        f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+                    )
             self._project = project_record
         if space is not None:
-            space_record = Space.filter(Q(name=space) | Q(uid=space)).one_or_none()
-            if space_record is None:
-                raise InvalidArgument(
-                    f"Space '{space}', please check on the hub UI whether you have the correct `uid` or `name`."
+            if isinstance(space, Space):
+                assert space._state.adding is False, (  # noqa: S101
+                    "Space must be saved before passing it to track()"
                 )
+                space_record = space
+            else:
+                space_record = Space.filter(Q(name=space) | Q(uid=space)).one_or_none()
+                if space_record is None:
+                    raise InvalidArgument(
+                        f"Space '{space}', please check on the hub UI whether you have the correct `uid` or `name`."
+                    )
             self._space = space_record
         self._logging_message_track = ""
         self._logging_message_imports = ""
         if transform is not None and isinstance(transform, str):
             self.uid = transform
             transform = None
+            uid_was_none = False
+        else:
+            uid_was_none = True
         self._path = None
         if transform is None:
             description = None
@@ -384,7 +391,14 @@ class Context:
             self._transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
-            new_run = False if self._transform.type == "notebook" else True  # type: ignore
+            new_run = (
+                False
+                if (
+                    self._transform.type == "notebook"
+                    and self._notebook_runner != "nbconvert"
+                )
+                else True
+            )  # type: ignore
 
         run = None
         if not new_run:  # try loading latest run by same user
@@ -431,6 +445,22 @@ class Context:
         logger.important(self._logging_message_track)
         if self._logging_message_imports:
             logger.important(self._logging_message_imports)
+        if uid_was_none:
+            notebook_or_script = (
+                "notebook" if self._transform.type == "notebook" else "script"
+            )
+            r_or_python = "."
+            if self._path is not None:
+                r_or_python = "." if self._path.suffix in {".py", ".ipynb"} else "$"
+            project_str = f', project="{project}"' if project is not None else ""
+            space_str = f', space="{space}"' if space is not None else ""
+            params_str = (
+                ", params={...}" if params is not None else ""
+            )  # do not put the values because typically parameterized by user
+            kwargs_str = f"{project_str}{space_str}{params_str}"
+            logger.important_hint(
+                f'recommendation: to identify the {notebook_or_script} across renames, pass the uid: ln{r_or_python}track("{self.transform.uid[:-4]}"{kwargs_str})'
+            )
 
     def _track_source_code(
         self,
@@ -472,7 +502,7 @@ class Context:
         path_str: str | None,
     ) -> tuple[Path, str | None]:
         if path_str is None:
-            path = get_notebook_path()
+            path, self._notebook_runner = get_notebook_path()
         else:
             path = Path(path_str)
         description = None
@@ -504,6 +534,53 @@ class Context:
                 pass
         return path, description
 
+    def _process_aux_transform(
+        self,
+        aux_transform: Transform,
+        transform_hash: str,
+    ) -> tuple[str, Transform | None, str]:
+        # first part of the if condition: no version bump, second part: version bump
+        message = ""
+        if (
+            # if a user hasn't yet saved the transform source code AND is the same user
+            (
+                aux_transform.source_code is None
+                and aux_transform.created_by_id == ln_setup.settings.user.id
+            )
+            # if the transform source code is unchanged
+            # if aux_transform.type == "notebook", we anticipate the user makes changes to the notebook source code
+            # in an interactive session, hence we *pro-actively bump* the version number by setting `revises` / 'nbconvert' execution is NOT interactive
+            # in the second part of the if condition even though the source code is unchanged at point of running track()
+            or (
+                aux_transform.hash == transform_hash
+                and (
+                    aux_transform.type != "notebook"
+                    or self._notebook_runner == "nbconvert"
+                )
+            )
+        ):
+            uid = aux_transform.uid
+            return uid, aux_transform, message
+        else:
+            uid = f"{aux_transform.uid[:-4]}{increment_base62(aux_transform.uid[-4:])}"
+            message = (
+                f"found {aux_transform.type} {aux_transform.key}, making new version"
+            )
+            if (
+                aux_transform.hash == transform_hash
+                and aux_transform.type == "notebook"
+            ):
+                message += " -- anticipating changes"
+            elif aux_transform.hash != transform_hash:
+                message += (
+                    ""  # could log "source code changed", but this seems too much
+                )
+            elif aux_transform.created_by_id != ln_setup.settings.user.id:
+                message += (
+                    f" -- {aux_transform.created_by.handle} already works on this draft"
+                )
+            return uid, None, message
+
     def _create_or_load_transform(
         self,
         *,
@@ -512,8 +589,22 @@ class Context:
         transform_ref_type: str | None = None,
         transform_type: TransformType = None,
     ):
-        # the user did not pass the uid
-        if self.uid is None:
+        from .._finish import notebook_to_script
+
+        if not self._path.suffix == ".ipynb":
+            transform_hash, _ = hash_file(self._path)
+        else:
+            # need to convert to stripped py:percent format for hashing
+            source_code_path = ln_setup.settings.cache_dir / self._path.name.replace(
+                ".ipynb", ".py"
+            )
+            notebook_to_script(description, self._path, source_code_path)
+            transform_hash, _ = hash_file(source_code_path)
+        # see whether we find a transform with the exact same hash
+        aux_transform = Transform.filter(hash=transform_hash).one_or_none()
+        # if the user did not pass a uid and there is no matching aux_transform
+        # need to search for the transform based on the filename
+        if self.uid is None and aux_transform is None:
 
             class SlashCount(Func):
                 template = "LENGTH(%(expressions)s) - LENGTH(REPLACE(%(expressions)s, '/', ''))"
@@ -528,47 +619,15 @@ class Context:
             uid = f"{base62_12()}0000"
             key = self._path.name
             target_transform = None
-            hash, _ = hash_file(self._path)
             if len(transforms) != 0:
                 message = ""
                 found_key = False
                 for aux_transform in transforms:
                     if aux_transform.key in self._path.as_posix():
                         key = aux_transform.key
-                        # first part of the if condition: no version bump, second part: version bump
-                        if (
-                            # if a user hasn't yet saved the transform source code, needs to be same user
-                            (
-                                aux_transform.source_code is None
-                                and aux_transform.created_by_id
-                                == ln_setup.settings.user.id
-                            )
-                            # if the transform source code is unchanged
-                            # if aux_transform.type == "notebook", we anticipate the user makes changes to the notebook source code
-                            # in an interactive session, hence we *pro-actively bump* the version number by setting `revises`
-                            # in the second part of the if condition even though the source code is unchanged at point of running track()
-                            or (
-                                aux_transform.hash == hash
-                                and aux_transform.type != "notebook"
-                            )
-                        ):
-                            uid = aux_transform.uid
-                            target_transform = aux_transform
-                        else:
-                            uid = f"{aux_transform.uid[:-4]}{increment_base62(aux_transform.uid[-4:])}"
-                            message = f"there already is a {aux_transform.type} with key '{aux_transform.key}'"
-                            if (
-                                aux_transform.hash == hash
-                                and aux_transform.type == "notebook"
-                            ):
-                                message += " -- anticipating changes"
-                            elif aux_transform.hash != hash:
-                                message += ""  # could log "source code changed", but this seems too much
-                            elif (
-                                aux_transform.created_by_id != ln_setup.settings.user.id
-                            ):
-                                message += f" -- {aux_transform.created_by.handle} already works on this draft"
-                            message += f", creating new version '{uid}'"
+                        uid, target_transform, message = self._process_aux_transform(
+                            aux_transform, transform_hash
+                        )
                         found_key = True
                         break
                 if not found_key:
@@ -584,7 +643,7 @@ class Context:
                     logger.important(message)
             self.uid, transform = uid, target_transform
         # the user did pass the uid
-        else:
+        elif self.uid is not None and len(self.uid) == 16:
             transform = Transform.filter(uid=self.uid).one_or_none()
             if transform is not None:
                 if transform.key not in self._path.as_posix():
@@ -599,6 +658,33 @@ class Context:
                     key = transform.key  # type: ignore
             else:
                 key = self._path.name
+        else:
+            if self.uid is not None:
+                assert len(self.uid) == 12, (  # noqa: S101
+                    "uid must be 12 (stem) or 16 (full) characters long"
+                )
+                aux_transform = (
+                    Transform.filter(uid__startswith=self.uid)
+                    .order_by("-created_at")
+                    .first()
+                )
+            if aux_transform is not None:
+                if aux_transform.key.endswith(self._path.name):
+                    key = aux_transform.key
+                else:
+                    key = "/".join(
+                        aux_transform.key.split("/")[:-1] + [self._path.name]
+                    )
+                uid, target_transform, message = self._process_aux_transform(
+                    aux_transform, transform_hash
+                )
+                if message != "":
+                    logger.important(message)
+            else:
+                uid = f"{self.uid}0000" if self.uid is not None else None
+                target_transform = None
+                key = self._path.name
+            self.uid, transform = uid, target_transform
         if self.version is not None:
             # test inconsistent version passed
             if (
@@ -610,15 +696,16 @@ class Context:
                     f"✗ please pass consistent version: ln.context.version = '{transform.version}'"  # type: ignore
                 )
             # test whether version was already used for another member of the family
-            suid, vuid = (self.uid[:-4], self.uid[-4:])
-            transform = Transform.filter(
-                uid__startswith=suid, version=self.version
-            ).one_or_none()
-            if transform is not None and vuid != transform.uid[-4:]:
-                better_version = bump_version_function(self.version)
-                raise SystemExit(
-                    f"✗ version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
-                )
+            if self.uid is not None and len(self.uid) == 16:
+                suid, vuid = (self.uid[:-4], self.uid[-4:])
+                transform = Transform.filter(
+                    uid__startswith=suid, version=self.version
+                ).one_or_none()
+                if transform is not None and vuid != transform.uid[-4:]:
+                    better_version = bump_version_function(self.version)
+                    raise SystemExit(
+                        f"✗ version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
+                    )
         # make a new transform record
         if transform is None:
             assert key is not None  # noqa: S101
@@ -659,13 +746,15 @@ class Context:
             # check whether transform source code was already saved
             if transform_was_saved:
                 bump_revision = False
-                if transform.type == "notebook":
+                if (
+                    transform.type == "notebook"
+                    and self._notebook_runner != "nbconvert"
+                ):
                     # we anticipate the user makes changes to the notebook source code
                     # in an interactive session, hence we pro-actively bump the version number
                     bump_revision = True
                 else:
-                    hash, _ = hash_file(self._path)  # ignore hash_type for now
-                    if hash != transform.hash:
+                    if transform_hash != transform.hash:
                         bump_revision = True
                     else:
                         self._logging_message_track += (
@@ -674,7 +763,10 @@ class Context:
                 if bump_revision:
                     change_type = (
                         "re-running notebook with already-saved source code"
-                        if transform.type == "notebook"
+                        if (
+                            transform.type == "notebook"
+                            and self._notebook_runner != "nbconvert"
+                        )
                         else "source code changed"
                     )
                     raise UpdateContext(
@@ -684,11 +776,11 @@ class Context:
                 self._logging_message_track += f"loaded Transform('{transform.uid}')"
         self._transform = transform
 
-    def finish(self, ignore_non_consecutive: None | bool = None) -> None:
-        """Finish a tracked run.
+    def _finish(self, ignore_non_consecutive: None | bool = None) -> None:
+        """Finish the run and write a run report.
 
         - writes a timestamp: `run.finished_at`
-        - saves the source code: `transform.source_code`
+        - saves the source code if it is not yet saved: `transform.source_code`
         - saves a run report: `run.report`
 
         When called in the last cell of a notebook:
@@ -732,6 +824,7 @@ class Context:
             finished_at=True,
             ignore_non_consecutive=ignore_non_consecutive,
             is_retry=self._is_finish_retry,
+            notebook_runner=self._notebook_runner,
         )
         if return_code == "retry":
             self._is_finish_retry = True
@@ -746,5 +839,13 @@ class Context:
         self._version = None
         self._description = None
 
+    @deprecated("ln.track()")
+    def track(self, *args, **kwargs):
+        return self._track(*args, **kwargs)
+
+    @deprecated("ln.finish()")
+    def finish(self, *args, **kwargs):
+        return self._finish(*args, **kwargs)
+
 
 context: Context = Context()

lamindb/core/_settings.py

@@ -9,6 +9,7 @@ from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core._settings import settings as setup_settings
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
 
+from .subsettings._annotation_settings import AnnotationSettings, annotation_settings
 from .subsettings._creation_settings import CreationSettings, creation_settings
 
 if TYPE_CHECKING:
@@ -34,13 +35,13 @@ VERBOSITY_TO_STR: dict[int, str] = dict(
 class Settings:
     """Settings.
 
-    Use ``lamindb.settings`` instead of instantiating this class yourself.
+    Use `lamindb.settings` instead of instantiating this class yourself.
     """
 
-    def __init__(self, git_repo: str | None):
+    def __init__(self):
         self._verbosity_int: int = 1  # warning-level logging
         logger.set_verbosity(self._verbosity_int)
-        self._sync_git_repo: str | None = git_repo
+        self._sync_git_repo: str | None = None
 
     @property
     def creation(self) -> CreationSettings:
@@ -51,6 +52,15 @@ class Settings:
         """
         return creation_settings
 
+    @property
+    def annotation(self) -> AnnotationSettings:
+        """Artifact annotation settings.
+
+        For example, `ln.settings.creation.search_names = False` will disable
+        searching for records with similar names during creation.
+        """
+        return annotation_settings
+
     track_run_inputs: bool = True
     """Track files as input upon `.load()`, `.cache()` and `.open()`.
 
@@ -85,13 +95,18 @@ class Settings:
 
         Provide the full git repo URL.
         """
-        return self._sync_git_repo
+        if self._sync_git_repo is not None:
+            return self._sync_git_repo
+        elif os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
+            return None
+        else:
+            return setup_settings.instance.git_repo
 
     @sync_git_repo.setter
     def sync_git_repo(self, value) -> None:
         """Sync transforms with scripts in git repository.
 
-        For example: `ln.sync_git_repo = https://github.com/laminlabs/redun-lamin`
+        For example: `ln.settings.sync_git_repo = https://github.com/laminlabs/redun-lamin`
         """
         self._sync_git_repo = sanitize_git_repo_url(value)
         if not self._sync_git_repo.startswith("https://"):  # pragma: nocover
@@ -99,28 +114,31 @@ class Settings:
 
     @property
     def storage(self) -> StorageSettings:
-        """Default storage location.
+        """Current default storage location for writes.
 
         Examples:
 
-        >>> ln.settings.storage
-        StorageSettings(root='s3://my-bucket', uid='j7MaPxtLxPeE')
+        Retrieve the storage settings::
+
+            ln.settings.storage
+            #> StorageSettings(root='s3://my-bucket')
 
-        >>> ln.settings.storage.root
-        UPath('s3://my-bucket')
+        Retrieve the storage root::
 
-        You can switch the default storage location to another managed storage
-        location by passing a string:
+            ln.settings.storage.root
+            #> UPath('s3://my-bucket')
 
-        >>> ln.settings.storage = "s3://some-bucket"
+        You can write artifacts to other storage locations by switching the current default storage location::
 
-        You can also pass additional fsspec kwargs via:
+            ln.settings.storage = "s3://some-bucket"
 
-        >>> kwargs = dict(
-        >>>     profile="some_profile", # fsspec arg
-        >>>     cache_regions=True # fsspec arg for s3
-        >>> )
-        >>> ln.settings.storage = "s3://some-bucket", kwargs
+        You can also pass additional fsspec kwargs via::
+
+            kwargs = dict(
+                profile="some_profile", # fsspec arg
+                cache_regions=True # fsspec arg for s3
+            )
+            ln.settings.storage = "s3://some-bucket", kwargs
         """
         return self._storage_settings
 
@@ -174,9 +192,4 @@ class Settings:
         logger.set_verbosity(verbosity_int)
 
 
-if os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
-    git_repo = None
-else:
-    git_repo = setup_settings.instance.git_repo
-
-settings = Settings(git_repo=git_repo)
+settings = Settings()

lamindb/core/datasets/__init__.py

@@ -1,12 +1,17 @@
 """Test datasets.
 
+The mini immuno dataset.
+
+.. autosummary::
+   :toctree: .
+
+   mini_immuno
+
 Small in-memory datasets.
 
 .. autosummary::
    :toctree: .
 
-   small_dataset1
-   small_dataset2
    anndata_with_obs
 
 Files.
@@ -59,6 +64,7 @@ Other.
    fake_bio_notebook_titles
 """
 
+from . import mini_immuno
 from ._core import (
     anndata_file_pbmc68k_test,
     anndata_human_immune_cells,
@@ -88,7 +94,8 @@ from ._core import (
 from ._fake import fake_bio_notebook_titles
 from ._small import (
     anndata_with_obs,
-    small_dataset1,
-    small_dataset2,
     small_dataset3_cellxgene,
 )
+
+small_dataset1 = mini_immuno.get_dataset1  # backward compat
+small_dataset2 = mini_immuno.get_dataset2  # backward compat