lamindb 1.6.2__py3-none-any.whl → 1.7.0__py3-none-any.whl

lamindb/__init__.py

@@ -46,7 +46,6 @@ Manage flexible records to track, e.g., samples or donors.
    :toctree: .
 
    Record
-   Sheet
 
 Manage projects.
 
@@ -109,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.6.2"
+__version__ = "1.7.0"
 
 import warnings
 
@@ -155,7 +154,6 @@ if _check_instance_setup(from_module="lamindb"):
         Space,
         Branch,
         Record,
-        Sheet,
     )
     from .models.save import save
     from . import core

lamindb/_finish.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamin_utils._logger import LEVEL_TO_COLORS, LEVEL_TO_ICONS, RESET_COLOR
-from lamindb_setup.core.hashing import hash_file
+from lamindb_setup.core.hashing import hash_dir, hash_file
 
 from lamindb.models import Artifact, Run, Transform
 
@@ -241,6 +241,7 @@ def save_context_core(
     transform: Transform,
     filepath: Path,
     finished_at: bool = False,
+    skip_save_report: bool = False,
     ignore_non_consecutive: bool | None = None,
     from_cli: bool = False,
     is_retry: bool = False,
@@ -343,7 +344,6 @@ def save_context_core(
         transform_hash, _ = hash_file(source_code_path)  # ignore hash_type for now
         if transform.hash is not None:
             # check if the hash of the transform source code matches
-            # (for scripts, we already run the same logic in track() - we can deduplicate the call at some point)
             if transform_hash != transform.hash:
                 response = input(
                     f"You are about to overwrite existing source code (hash '{transform.hash}') for Transform('{transform.uid}')."
@@ -361,26 +361,46 @@ def save_context_core(
             transform.source_code = source_code_path.read_text()
             transform.hash = transform_hash
 
-    # track run environment
     if run is not None:
-        env_path = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
-        if env_path.exists():
+        base_path = ln_setup.settings.cache_dir / "environments" / f"run_{run.uid}"
+        paths = [base_path / "run_env_pip.txt", base_path / "r_pak_lockfile.json"]
+        existing_paths = [path for path in paths if path.exists()]
+
+        if existing_paths:
             overwrite_env = True
             if run.environment_id is not None and from_cli:
                 logger.important("run.environment is already saved, ignoring")
                 overwrite_env = False
+
             if overwrite_env:
-                env_hash, _ = hash_file(env_path)
+                # Use directory if multiple files exist, otherwise use the single file
+                artifact_path: Path = (
+                    base_path if len(existing_paths) > 1 else existing_paths[0]
+                )
+
+                # Set description based on what we're saving
+                if len(existing_paths) == 1:
+                    if existing_paths[0].name == "run_env_pip.txt":
+                        description = "requirements.txt"
+                    elif existing_paths[0].name == "r_pak_lockfile.json":
+                        description = "r_pak_lockfile.json"
+                    env_hash, _ = hash_file(artifact_path)
+                else:
+                    description = "environments"
+                    _, env_hash, _, _ = hash_dir(artifact_path)
+
                 artifact = ln.Artifact.objects.filter(hash=env_hash).one_or_none()
                 new_env_artifact = artifact is None
+
                 if new_env_artifact:
-                    artifact = ln.Artifact(  # type: ignore
-                        env_path,
-                        description="requirements.txt",
+                    artifact = ln.Artifact(
+                        artifact_path,
+                        description=description,
                         kind="__lamindb_run__",
                         run=False,
                     )
                     artifact.save(upload=True, print_progress=False)
+
                 run.environment = artifact
                 if new_env_artifact:
                     logger.debug(f"saved run.environment: {run.environment}")
@@ -394,12 +414,8 @@ def save_context_core(
         if update_finished_at:
             run.finished_at = datetime.now(timezone.utc)
 
-    # track logs
-    if run is not None and not from_cli and not is_ipynb and not is_r_notebook:
-        save_run_logs(run)
-
     # track report and set is_consecutive
-    if save_source_code_and_report:
+    if save_source_code_and_report and not skip_save_report:
         if run is not None:
             # do not save a run report if executing through nbconvert
             if report_path is not None and notebook_runner != "nbconvert":
@@ -454,7 +470,7 @@ def save_context_core(
         ln.Transform.get(transform_id_prior_to_save).delete()
 
     # finalize
-    if not from_cli and run is not None:
+    if finished_at and not from_cli and run is not None:
         run_time = run.finished_at - run.started_at
         days = run_time.days
         seconds = run_time.seconds
@@ -480,7 +496,7 @@ def save_context_core(
             logger.important(
                 f"go to: https://lamin.ai/{instance_slug}/transform/{transform.uid}"
             )
-        if not from_cli and save_source_code_and_report:
+        if finished_at and not from_cli and save_source_code_and_report:
             thing = "notebook" if (is_ipynb or is_r_notebook) else "script"
             logger.important(
                 f"to update your {thing} from the CLI, run: lamin save {filepath}"

lamindb/base/types.py

@@ -27,7 +27,7 @@ from typing import Literal, Union
 import numpy as np
 import pandas as pd
 from django.db.models.query_utils import DeferredAttribute as FieldAttr
-from lamindb_setup.core.types import UPathStr  # noqa: F401
+from lamindb_setup.types import UPathStr  # noqa: F401
 
 # need to use Union because __future__.annotations doesn't do the job here <3.10
 # typing.TypeAlias, >3.10 on but already deprecated
@@ -37,7 +37,7 @@ StrField = Union[str, FieldAttr]  # typing.TypeAlias
 TransformType = Literal[
     "pipeline", "notebook", "upload", "script", "function", "linker"
 ]
-ArtifactKind = Literal["dataset", "model"]
+ArtifactKind = Literal["dataset", "model", "__lamindb_run__"]
 
 # below is used for Feature.dtype and Param.dtype
 Dtype = Literal[
@@ -51,10 +51,11 @@ Dtype = Literal[
     "datetime",  # datetime
     "dict",  # dictionary
     "object",  # this is a pandas input dtype, we're only using it for complicated types, not for strings
+    "path",  # path, validated as str, but specially treated in the UI
 ]
 """Data type.
 
-Data types in lamindb are a string-serialized abstraction of common data types.
+String-serialized representations of common data types.
 
 Overview
 ========
@@ -68,8 +69,9 @@ integer       `"int"`       `int64 | int32 | int16 | int8 | uint | ...`
 float         `"float"`     `float64 | float32 | float16 | float8 | ...`
 string        `"str"`       `object`
 datetime      `"datetime"`  `datetime`
-date          `"date"`      `date`
+date          `"date"`      `object` (pandera requires an ISO-format string, convert with `df["date"] = df["date"].dt.date`)
 dictionary    `"dict"`      `object`
+path          `"path"`      `str` (pandas does not have a dedicated path type, validated as `str`)
 ============  ============  =================================================
 
 Categoricals

lamindb/core/_context.py

@@ -9,7 +9,7 @@ import threading
 import traceback
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TextIO
 
 import lamindb_setup as ln_setup
 from django.db.models import Func, IntegerField, Q
@@ -32,10 +32,10 @@ from ..models._is_versioned import (
     increment_base62,
 )
 from ._sync_git import get_transform_reference_from_git_repo
-from ._track_environment import track_environment
+from ._track_environment import track_python_environment
 
 if TYPE_CHECKING:
-    from lamindb_setup.core.types import UPathStr
+    from lamindb_setup.types import UPathStr
 
     from lamindb.base.types import TransformType
     from lamindb.models import Branch, Project, Space
@@ -100,19 +100,50 @@ def pretty_pypackages(dependencies: dict) -> str:
     return " ".join(deps_list)
 
 
+def last_non_empty_r_block(line: str) -> str:
+    for block in reversed(line.split("\r")):
+        if block:
+            return block
+    return ""
+
+
 class LogStreamHandler:
-    def __init__(self, log_stream, file):
+    def __init__(self, log_stream: TextIO, file: TextIO, use_buffer: bool):
         self.log_stream = log_stream
         self.file = file
 
-    def write(self, data):
+        self._buffer = ""
+        self._use_buffer = use_buffer
+
+    def write(self, data: str) -> int:
         self.log_stream.write(data)
-        self.file.write(data)
-        self.file.flush()
+
+        if not self._use_buffer:
+            self.file.write(data)
+            self.file.flush()
+            return len(data)
+
+        self._buffer += data
+        # write only the last part of a line with carriage returns
+        while "\n" in self._buffer:
+            line, self._buffer = self._buffer.split("\n", 1)
+            self.file.write(last_non_empty_r_block(line) + "\n")
+            self.file.flush()
+
+        return len(data)
 
     def flush(self):
         self.log_stream.flush()
-        self.file.flush()
+        if not self.file.closed:
+            self.file.flush()
+
+    # .flush is sometimes (in jupyter etc.) called after every .write
+    # this needs to be called only at the end
+    def flush_buffer(self):
+        if not self.file.closed and self._buffer:
+            self.file.write(last_non_empty_r_block(self._buffer))
+            self._buffer = ""
+        self.flush()
 
 
 class LogStreamTracker:
@@ -131,8 +162,14 @@ class LogStreamTracker:
             ln_setup.settings.cache_dir / f"run_logs_{self.run.uid}.txt"
         )
         self.log_file = open(self.log_file_path, "w")
-        sys.stdout = LogStreamHandler(self.original_stdout, self.log_file)
-        sys.stderr = LogStreamHandler(self.original_stderr, self.log_file)
+        # use buffering for correct handling of carriage returns
+        sys.stdout = LogStreamHandler(
+            self.original_stdout, self.log_file, use_buffer=True
+        )
+        # write evrything immediately in stderr
+        sys.stderr = LogStreamHandler(
+            self.original_stderr, self.log_file, use_buffer=False
+        )
         # handle signals
         # signal should be used only in the main thread, otherwise
         # ValueError: signal only works in main thread of the main interpreter
@@ -144,43 +181,60 @@ class LogStreamTracker:
 
     def finish(self):
         if self.original_stdout:
+            getattr(sys.stdout, "flush_buffer", sys.stdout.flush)()
+            sys.stderr.flush()
             sys.stdout = self.original_stdout
             sys.stderr = self.original_stderr
             self.log_file.close()
 
     def cleanup(self, signo=None, frame=None):
-        from lamindb._finish import save_run_logs
-
-        if self.original_stdout and not self.is_cleaning_up:
-            self.is_cleaning_up = True
-            if signo is not None:
-                signal_msg = f"\nProcess terminated by signal {signo} ({signal.Signals(signo).name})\n"
-                if frame:
-                    signal_msg += (
-                        f"Frame info:\n{''.join(traceback.format_stack(frame))}"
-                    )
-                self.log_file.write(signal_msg)
-            sys.stdout = self.original_stdout
-            sys.stderr = self.original_stderr
-            self.log_file.flush()
-            self.log_file.close()
-            save_run_logs(self.run, save_run=True)
+        try:
+            from .._finish import save_run_logs
+
+            if self.original_stdout and not self.is_cleaning_up:
+                self.is_cleaning_up = True
+                getattr(sys.stdout, "flush_buffer", sys.stdout.flush)()
+                sys.stderr.flush()
+                if signo is not None:
+                    signal_msg = f"\nProcess terminated by signal {signo} ({signal.Signals(signo).name})\n"
+                    if frame:
+                        signal_msg += (
+                            f"Frame info:\n{''.join(traceback.format_stack(frame))}"
+                        )
+                    self.log_file.write(signal_msg)
+                    self.log_file.flush()
+                    self.run._status_code = 2  # aborted
+                else:
+                    self.run._status_code = 1  # errored
+                self.run.finished_at = datetime.now(timezone.utc)
+                sys.stdout = self.original_stdout
+                sys.stderr = self.original_stderr
+                self.log_file.close()
+                save_run_logs(self.run, save_run=True)
+        except:  # noqa: E722, S110
+            pass
 
     def handle_exception(self, exc_type, exc_value, exc_traceback):
-        if not self.is_cleaning_up:
-            error_msg = f"{''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))}"
-            if self.log_file.closed:
-                self.log_file = open(self.log_file_path, "a")
-            self.log_file.write(error_msg)
-            self.log_file.flush()
-            self.cleanup()
+        try:
+            if not self.is_cleaning_up:
+                error_msg = f"{''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))}"
+                if self.log_file.closed:
+                    self.log_file = open(self.log_file_path, "a")
+                else:
+                    getattr(sys.stdout, "flush_buffer", sys.stdout.flush)()
+                    sys.stderr.flush()
+                self.log_file.write(error_msg)
+                self.log_file.flush()
+                self.cleanup()
+        except:  # noqa: E722, S110
+            pass
         self.original_excepthook(exc_type, exc_value, exc_traceback)
 
 
 class Context:
     """Run context.
 
-    Is the book keeper for :meth:`~lamindb.core.Context.track`.
+    Is the book keeper for :func:`~lamindb.track` and :func:`~lamindb.finish`.
     """
 
     def __init__(self):
@@ -190,7 +244,6 @@ class Context:
         self._transform: Transform | None = None
         self._run: Run | None = None
         self._path: Path | None = None
-        """A local path to the script or notebook that's running."""
         self._project: Project | None = None
         self._space: Space | None = None
         self._branch: Branch | None = None
@@ -281,8 +334,8 @@ class Context:
         Args:
             transform: A transform (stem) `uid` (or record). If `None`, auto-creates a `transform` with its `uid`.
             project: A project (or its `name` or `uid`) for labeling entities.
-            space: A restricted space (or its `name` or `uid`) in which to store artifacts, collections, transforms, and runs.
-                Default: the `"All"` space.
+            space: A restricted space (or its `name` or `uid`) in which to store entities.
+                Default: the `"all"` space. Note that bionty entities ignore this setting and always get written to the `"all"` space.
                 If you want to manually move entities to a different space, set the `.space` field (:doc:`docs:access`).
             branch: A branch (or its `name` or `uid`) on which to store records.
             params: A dictionary of parameters to track for the run.
@@ -308,6 +361,10 @@ class Context:
         """
         from lamindb.models import Branch, Project, Space
 
+        from .._finish import (
+            save_context_core,
+        )
+
         instance_settings = ln_setup.settings.instance
         # similar logic here: https://github.com/laminlabs/lamindb/pull/2527
         # TODO: refactor upon new access management
@@ -433,6 +490,7 @@ class Context:
             )
             if run is not None:  # loaded latest run
                 run.started_at = datetime.now(timezone.utc)  # update run time
+                run._status_code = -2  # re-started
                 self._logging_message_track += f", re-started Run('{run.uid[:8]}...') at {format_field_value(run.started_at)}"
 
         if run is None:  # create new run
@@ -441,6 +499,7 @@ class Context:
                 params=params,
             )
             run.started_at = datetime.now(timezone.utc)
+            run._status_code = -1  # started
             self._logging_message_track += f", started new Run('{run.uid[:8]}...') at {format_field_value(run.started_at)}"
         # can only determine at ln.finish() if run was consecutive in
         # interactive session, otherwise, is consecutive
@@ -453,7 +512,7 @@ class Context:
                 f"{key}={value}" for key, value in params.items()
             )
         self._run = run
-        track_environment(run)
+        track_python_environment(run)
         if self.project is not None:
             # to update a potential project link
             # is only necessary if transform is loaded rather than newly created
@@ -491,6 +550,8 @@ class Context:
             logger.important_hint(
                 f'recommendation: to identify the {notebook_or_script} across renames, pass the uid: ln{r_or_python}track("{self.transform.uid[:-4]}"{kwargs_str})'
             )
+        if self.transform.type == "script":
+            save_context_core(run=run, transform=self.transform, filepath=self._path)
 
     def _track_source_code(
         self,
@@ -653,6 +714,8 @@ class Context:
                 message = ""
                 found_key = False
                 for aux_transform in transforms:
+                    # check whether the transform key is in the path
+                    # that's not going to be the case for keys that have "/" in them and don't match the folder
                     if aux_transform.key in self._path.as_posix():
                         key = aux_transform.key
                         uid, target_transform, message = self._process_aux_transform(
@@ -668,7 +731,7 @@ class Context:
                             for transform in transforms
                         ]
                     )
-                    message = f"ignoring transform{plural_s} with same filename:\n{transforms_str}"
+                    message = f"ignoring transform{plural_s} with same filename in different folder:\n{transforms_str}"
                 if message != "":
                     logger.important(message)
             self.uid, transform = uid, target_transform
@@ -715,7 +778,10 @@ class Context:
                         f"Please respond with either 1 or 2, not {response}"
                     )
                     if response == "2":
-                        transform_hash = None  # make a new transform
+                        aux_transform, transform_hash = (
+                            None,
+                            None,
+                        )  # make a new transform
             if aux_transform is not None:
                 if aux_transform.key.endswith(self._path.name):
                     key = aux_transform.key
@@ -740,8 +806,9 @@ class Context:
                 and transform.version is not None  # type: ignore
                 and self.version != transform.version  # type: ignore
             ):
-                raise SystemExit(
-                    f"✗ please pass consistent version: ln.context.version = '{transform.version}'"  # type: ignore
+                raise ValueError(
+                    f"Transform is already tagged with version {transform.version}, but you passed {self.version}\n"  # noqa: S608
+                    f"If you want to update the transform version, set it outside ln.track(): transform.version = '{self.version}'; transform.save()"
                 )
             # test whether version was already used for another member of the family
             if self.uid is not None and len(self.uid) == 16:
@@ -850,9 +917,7 @@ class Context:
             `lamin save script.py` or `lamin save notebook.ipynb` → `docs </cli#lamin-save>`__
 
         """
-        from lamindb._finish import (
-            save_context_core,
-        )
+        from .._finish import save_context_core, save_run_logs
 
         if self.run is None:
             raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
@@ -865,18 +930,23 @@ class Context:
             self.run.save()
             # nothing else to do
             return None
-        return_code = save_context_core(
-            run=self.run,
-            transform=self.run.transform,
-            filepath=self._path,
-            finished_at=True,
-            ignore_non_consecutive=ignore_non_consecutive,
-            is_retry=self._is_finish_retry,
-            notebook_runner=self._notebook_runner,
-        )
-        if return_code == "retry":
-            self._is_finish_retry = True
-            return None
+        self.run._status_code = 0
+        if self.transform.type == "notebook":
+            return_code = save_context_core(
+                run=self.run,
+                transform=self.run.transform,
+                filepath=self._path,
+                finished_at=True,
+                ignore_non_consecutive=ignore_non_consecutive,
+                is_retry=self._is_finish_retry,
+                notebook_runner=self._notebook_runner,
+            )
+            if return_code == "retry":
+                self._is_finish_retry = True
+                return None
+        else:
+            self.run.finished_at = datetime.now(timezone.utc)
+            save_run_logs(self.run, save_run=True)
         if self.transform.type != "notebook":
             self._stream_tracker.finish()
         # reset the context attributes so that somebody who runs `track()` after finish

lamindb/core/_mapped_collection.py

@@ -21,7 +21,7 @@ from .storage._anndata_accessor import (
 )
 
 if TYPE_CHECKING:
-    from lamindb_setup.core.types import UPathStr
+    from lamindb_setup.types import UPathStr
 
 
 class _Connect:

lamindb/core/_settings.py

@@ -4,7 +4,7 @@ import os
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
-from lamin_utils import logger
+from lamin_utils import colors, logger
 from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core._settings import settings as setup_settings
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
@@ -35,7 +35,7 @@ VERBOSITY_TO_STR: dict[int, str] = dict(
 class Settings:
     """Settings.
 
-    Use `lamindb.settings` instead of instantiating this class yourself.
+    Please use the global `ln.settings` object instead of instantiating this class yourself.
     """
 
     def __init__(self):
@@ -43,6 +43,36 @@ class Settings:
         logger.set_verbosity(self._verbosity_int)
         self._sync_git_repo: str | None = None
 
+    def __repr__(self) -> str:  # pragma: no cover
+        cls_name = colors.green(self.__class__.__name__)
+        verbosity_color = colors.yellow if self.verbosity == "warning" else colors.green
+        verbosity_str = verbosity_color(self.verbosity)
+
+        storage_root = self._storage_settings.root_as_str
+        storage_str = colors.italic(storage_root)
+
+        instance_str = colors.italic(self.instance_uid)
+        track_color = colors.green if self.track_run_inputs else colors.yellow
+        track_str = track_color(str(self.track_run_inputs))
+
+        lines = [
+            f"{cls_name}",
+            f"  instance: {instance_str}",
+            f"  storage: {storage_str}",
+            f"  verbosity: {verbosity_str}",
+            f"  track_run_inputs: {track_str}",
+        ]
+
+        if self.sync_git_repo:
+            repo_name = (
+                self.sync_git_repo.split("/")[-1]
+                if "/" in self.sync_git_repo
+                else self.sync_git_repo
+            )
+            lines.append(f"  sync_git_repo: {colors.italic(repo_name)}")
+
+        return "\n".join(lines)
+
     @property
     def creation(self) -> CreationSettings:
         """SQLRecord creation settings.
@@ -61,10 +91,15 @@ class Settings:
         """
         return annotation_settings
 
+    # note: this setting should probably be deprecated soon
+    # warnings could then be filtered with a regular warning mechanism
     track_run_inputs: bool = True
-    """Track files as input upon `.load()`, `.cache()` and `.open()`.
+    """Track run inputs (default `True`).
 
-    Requires a global run context with :func:`~lamindb.core.Context.track` was created!
+    If this setting is true, an artifact is recorded as run input upon `.load()`, `.cache()` & `.open()` provided :func:`~lamindb.track` was called in the current compute (Python, R) session.
+    If :func:`~lamindb.track` was not called, you receive a warning message upon `.load()`, `.cache()` & `.open()`.
+
+    If you switch this setting to `False`, you won't see the warning message anymore and no run inputs will be recorded.
 
     FAQ: :doc:`/faq/track-run-inputs`
     """
@@ -150,6 +185,11 @@ class Settings:
             path, kwargs = path_kwargs, {}
         set_managed_storage(path, **kwargs)
 
+    @property
+    def instance_uid(self) -> str:
+        """The `uid` of the current instance."""
+        return ln_setup.settings.instance.uid
+
     @property
     def cache_dir(self) -> UPath:
         """Cache root, a local directory to cache cloud files."""

lamindb/core/_track_environment.py

@@ -11,8 +11,11 @@ if TYPE_CHECKING:
     from lamindb.models import Run
 
 
-def track_environment(run: Run) -> None:
-    filepath = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
+def track_python_environment(run: Run) -> None:
+    env_dir = ln_setup.settings.cache_dir / "environments" / f"run_{run.uid}"
+    filepath = env_dir / "run_env_pip.txt"
+    if not env_dir.exists():
+        filepath.parent.mkdir(parents=True)
     # create a requirements.txt
     # we don't create a conda environment.yml mostly for its slowness
     try:

lamindb/core/loaders.py

@@ -34,7 +34,7 @@ from ..core._settings import settings
 
 if TYPE_CHECKING:
     from anndata import AnnData
-    from lamindb_setup.core.types import UPathStr
+    from lamindb_setup.types import UPathStr
     from mudata import MuData
 
     from lamindb.core.types import ScverseDataStructures

lamindb/core/storage/_anndata_accessor.py

@@ -26,7 +26,7 @@ if TYPE_CHECKING:
     from collections.abc import Mapping
 
     from fsspec.core import OpenFile
-    from lamindb_setup.core.types import UPathStr
+    from lamindb_setup.types import UPathStr
 
 
 anndata_version_parse = version.parse(anndata_version)