lamindb 0.75.1__py3-none-any.whl → 0.76.1__py3-none-any.whl

lamindb/__init__.py

@@ -21,9 +21,8 @@ Key functionality:
 .. autosummary::
    :toctree: .
 
+   context
    connect
-   track
-   finish
    Curate
    view
    save
@@ -42,7 +41,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.75.1"
+__version__ = "0.76.1"
 
 import os as _os
 
@@ -93,10 +92,9 @@ if _check_instance_setup(from_lamindb=True):
         integrations,
     )
     from ._curate import Curate
-    from ._finish import finish
     from ._save import save
     from ._view import view
-    from .core._run_context import run_context as _run_context
+    from .core._context import context
     from .core._settings import settings
 
     # schema modules
@@ -107,8 +105,8 @@ if _check_instance_setup(from_lamindb=True):
 
         _reload_schema_modules(_lamindb_setup.settings.instance)
 
-    track = _run_context._track
+    track = context.track  # backward compat
+    finish = context.finish  # backward compat
     settings.__doc__ = """Global :class:`~lamindb.core.Settings`."""
+    context.__doc__ = """Global :class:`~lamindb.core.Context`."""
     from django.db.models import Q
-
-    Annotate = Curate  # backward compat

lamindb/_artifact.py

@@ -107,7 +107,7 @@ def process_pathlike(
                 new_root = list(filepath.parents)[-1]
                 # do not register remote storage locations on hub if the current instance
                 # is not managed on the hub
-                storage_settings = init_storage(
+                storage_settings, _ = init_storage(
                     new_root, prevent_register_hub=not setup_settings.instance.is_on_hub
                 )
                 storage_record = register_storage_in_instance(storage_settings)
@@ -594,6 +594,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
     kwargs["description"] = description
     kwargs["visibility"] = visibility
     kwargs["_accessor"] = accessor
+    kwargs["is_new_version_of"] = is_new_version_of
     # this check needs to come down here because key might be populated from an
     # existing file path during get_artifact_kwargs_from_data()
     if (

lamindb/_collection.py

@@ -158,6 +158,7 @@ def __init__(
             run=run,
             version=version,
             visibility=visibility,
+            is_new_version_of=is_new_version_of,
             **kwargs,
         )
     collection._artifacts = artifacts

lamindb/_curate.py

@@ -313,11 +313,11 @@ class DataFrameCurator:
 
     def clean_up_failed_runs(self):
         """Clean up previous failed runs that don't save any outputs."""
-        from lamindb.core._run_context import run_context
+        from lamindb.core._context import context
 
-        if run_context.transform is not None:
-            Run.filter(transform=run_context.transform, output_artifacts=None).exclude(
-                uid=run_context.run.uid
+        if context.run is not None:
+            Run.filter(transform=context.run.transform, output_artifacts=None).exclude(
+                uid=context.run.uid
             ).delete()
 
 

lamindb/_feature.py

@@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs):
         return None
     # now we proceed with the user-facing constructor
     if len(args) != 0:
-        raise ValueError("Only non-keyword args allowed")
+        raise ValueError("Only keyword args allowed")
     dtype: type | str = kwargs.pop("dtype") if "dtype" in kwargs else None
     # cast type
     if dtype is None:

lamindb/_finish.py

@@ -9,9 +9,6 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
-from lnschema_core.types import TransformType
-
-from .core._run_context import is_run_from_ipython, run_context
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -21,51 +18,7 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 
-class TrackNotCalled(SystemExit):
-    pass
-
-
-class NotebookNotSaved(SystemExit):
-    pass
-
-
-def get_seconds_since_modified(filepath) -> float:
-    return datetime.now().timestamp() - filepath.stat().st_mtime
-
-
-def finish() -> None:
-    """Mark a tracked run as finished.
-
-    Saves source code and, for notebooks, a run report to your default storage location.
-    """
-    if run_context.run is None:
-        raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
-    if run_context.path is None:
-        if run_context.transform.type in {"script", "notebook"}:
-            raise ValueError(
-                f"Transform type is not allowed to be 'script' or 'notebook' but is {run_context.transform.type}."
-            )
-        run_context.run.finished_at = datetime.now(timezone.utc)
-        run_context.run.save()
-        # nothing else to do
-        return None
-    if is_run_from_ipython:  # notebooks
-        if (
-            get_seconds_since_modified(run_context.path) > 3
-            and os.getenv("LAMIN_TESTING") is None
-        ):
-            raise NotebookNotSaved(
-                "Please save the notebook in your editor right before running `ln.finish()`"
-            )
-    save_run_context_core(
-        run=run_context.run,
-        transform=run_context.transform,
-        filepath=run_context.path,
-        finished_at=True,
-    )
-
-
-def save_run_context_core(
+def save_context_core(
     *,
     run: Run,
     transform: Transform,
@@ -76,6 +29,8 @@ def save_run_context_core(
 ) -> str | None:
     import lamindb as ln
 
+    from .core._context import context, is_run_from_ipython
+
     ln.settings.verbosity = "success"
 
     # for scripts, things are easy
@@ -182,7 +137,9 @@ def save_run_context_core(
                     f"replaced transform._source_code_artifact: {transform._source_code_artifact}"
                 )
             else:
-                logger.warning("Please re-run `ln.track()` to make a new version")
+                logger.warning(
+                    "Please re-run `ln.context.track()` to make a new version"
+                )
                 return "rerun-the-notebook"
         else:
             logger.important("source code is already saved")
@@ -282,7 +239,7 @@ def save_run_context_core(
             logger.important(
                 f"if you want to update your {thing} without re-running it, use `lamin save {name}`"
             )
-    # because run & transform changed, update the global run_context
-    run_context.run = run
-    run_context.transform = transform
+    # because run & transform changed, update the global context
+    context._run = run
+    context._transform = transform
     return None

lamindb/_from_values.py

@@ -47,15 +47,35 @@ def get_or_create_records(
 
         # new records to be created based on new values
         if len(nonexist_values) > 0:
-            if source:
-                from_source = not source.in_db
-            elif (
-                records
-                and hasattr(records[0], "source_id")
-                and records[0].source_id
-                and records[0].source.in_db
-            ):
+            source_record = None
+            if from_source:
+                if isinstance(source, Record):
+                    source_record = source
+                elif (
+                    len(records) > 0
+                    and hasattr(records[0], "source_id")
+                    and records[0].source_id
+                ):
+                    source_record = records[0].source
+            if not source_record and hasattr(Record, "public"):
+                from bionty._bionty import get_source_record
+
+                source_record = get_source_record(Record.public(organism=organism))
+            if source_record:
+                from bionty.core._add_ontology import check_source_in_db
+
+                check_source_in_db(
+                    registry=Record,
+                    source=source_record,
+                    update=True,
+                )
+
+                from_source = not source_record.in_db
+            elif hasattr(Record, "source_id"):
+                from_source = True
+            else:
                 from_source = False
+
             if from_source:
                 records_bionty, unmapped_values = create_records_from_source(
                     iterable_idx=nonexist_values,
@@ -211,10 +231,6 @@ def create_records_from_source(
         return records, iterable_idx
     # add source record to the kwargs
     source_record = get_source_record(public_ontology)
-    if source_record is not None and source_record.in_db:
-        # skips the creation of records from public if the source is already in the db
-        return records, iterable_idx
-
     kwargs.update({"source": source_record})
 
     # filter the columns in bionty df based on fields

lamindb/_query_manager.py

@@ -40,11 +40,11 @@ class QueryManager(models.Manager):
                 self.source_field_name == "collection"
                 and self.target_field_name == "artifact"
             ):
+                from lamindb.core._context import context
                 from lamindb.core._data import WARNING_RUN_TRANSFORM, _track_run_input
-                from lamindb.core._run_context import run_context
 
                 if (
-                    run_context.run is None
+                    context.run is None
                     and not settings.creation.artifact_silence_missing_run_warning
                 ):
                     logger.warning(WARNING_RUN_TRANSFORM)

lamindb/_query_set.py

@@ -17,14 +17,12 @@ from lnschema_core.models import (
     Transform,
 )
 
+from lamindb.core.exceptions import DoesNotExist
+
 if TYPE_CHECKING:
     from lnschema_core.types import ListLike, StrField
 
 
-class NoResultFound(Exception):
-    pass
-
-
 class MultipleResultsFound(Exception):
     pass
 
@@ -59,7 +57,7 @@ def get_keys_from_df(data: list, registry: Record) -> list[str]:
 
 def one_helper(self):
     if len(self) == 0:
-        raise NoResultFound
+        raise DoesNotExist
     elif len(self) > 1:
         raise MultipleResultsFound(self)
     else:
@@ -243,10 +241,10 @@ class QuerySet(models.QuerySet, CanValidate):
         else:
             raise MultipleResultsFound(self.all())
 
-    def latest_version(self) -> RecordsList:
+    def latest_version(self) -> QuerySet:
         """Filter every version family by latest version."""
         if issubclass(self.model, IsVersioned):
-            return filter_query_set_by_latest_version(self)
+            return self.filter(is_latest=True)
         else:
             raise ValueError("Record isn't subclass of `lamindb.core.IsVersioned`")
 
@@ -288,29 +286,6 @@ class QuerySet(models.QuerySet, CanValidate):
         return _standardize(cls=self, values=values, field=field, **kwargs)
 
 
-def filter_query_set_by_latest_version(ordered_query_set: QuerySet) -> RecordsList:
-    # evaluating length can be very costly, hence, the try-except block
-    try:
-        first_record = ordered_query_set[0]
-    except IndexError:
-        return ordered_query_set
-    records_in_view = {}
-    records_in_view[first_record.stem_uid] = first_record
-    for record in ordered_query_set:
-        # this overwrites user-provided ordering (relevant records ordered by a
-        # certain field will not show if they are not the latest version)
-        if record.stem_uid not in records_in_view:
-            records_in_view[record.stem_uid] = record
-        else:
-            if record.created_at > records_in_view[record.stem_uid].created_at:
-                # deleting the entry is needed to preserve the integrity of
-                # user-provided ordering
-                del records_in_view[record.stem_uid]
-                records_in_view[record.stem_uid] = record
-    list_records_in_view = RecordsList(records_in_view.values())
-    return list_records_in_view
-
-
 models.QuerySet.df = QuerySet.df
 models.QuerySet.list = QuerySet.list
 models.QuerySet.first = QuerySet.first

lamindb/_record.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, NamedTuple
 
 import dj_database_url
 import lamindb_setup as ln_setup
-from django.db import connections
+from django.db import connections, transaction
 from django.db.models import IntegerField, Manager, Q, QuerySet, Value
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
@@ -123,24 +123,38 @@ def filter(cls, **expressions) -> QuerySet:
 
 @classmethod  # type:ignore
 @doc_args(Record.get.__doc__)
-def get(cls, idlike: int | str) -> Record:
+def get(
+    cls,
+    idlike: int | str | None = None,
+    **expressions,
+) -> Record:
     """{}"""  # noqa: D415
     from lamindb._filter import filter
 
     if isinstance(idlike, int):
         return filter(cls, id=idlike).one()
-    else:
+    elif isinstance(idlike, str):
         qs = filter(cls, uid__startswith=idlike)
         if issubclass(cls, IsVersioned):
-            return qs.latest_version().one()
+            if len(idlike) <= cls._len_stem_uid:
+                return qs.latest_version().one()
+            else:
+                return qs.one()
         else:
             return qs.one()
+    else:
+        assert idlike is None  # noqa: S101
+        # below behaves exactly like `.one()`
+        return cls.objects.get(**expressions)
 
 
 @classmethod  # type:ignore
 @doc_args(Record.df.__doc__)
 def df(
-    cls, include: str | list[str] | None = None, join: str = "inner"
+    cls,
+    include: str | list[str] | None = None,
+    join: str = "inner",
+    limit: int = 100,
 ) -> pd.DataFrame:
     """{}"""  # noqa: D415
     from lamindb._filter import filter
@@ -148,7 +162,7 @@ def df(
     query_set = filter(cls)
     if hasattr(cls, "updated_at"):
         query_set = query_set.order_by("-updated_at")
-    return query_set.df(include=include, join=join)
+    return query_set[:limit].df(include=include, join=join)
 
 
 # from_values doesn't apply for QuerySet or Manager
@@ -165,9 +179,7 @@ def from_values(
 ) -> list[Record]:
     """{}"""  # noqa: D415
     from_source = True if cls.__module__.startswith("bionty.") else False
-    # if records from source is already saved in db, skip from_source
-    if isinstance(source, Record) and source.in_db:
-        from_source = False
+
     field_str = get_name_field(cls, field=field)
     return get_or_create_records(
         iterable=values,
@@ -472,8 +484,8 @@ def transfer_to_default_db(
             return record_on_default
         if not mute:
             logger.hint(f"saving from instance {db} to default instance: {record}")
+        from lamindb.core._context import context
         from lamindb.core._data import WARNING_RUN_TRANSFORM
-        from lamindb.core._run_context import run_context
 
         if hasattr(record, "created_by_id"):
             # this line is needed to point created_by to default db
@@ -481,16 +493,16 @@ def transfer_to_default_db(
             record.created_by_id = ln_setup.settings.user.id
         if hasattr(record, "run_id"):
             record.run = None
-            if run_context.run is not None:
-                record.run_id = run_context.run.id
+            if context.run is not None:
+                record.run_id = context.run.id
             else:
                 if not settings.creation.artifact_silence_missing_run_warning:
                     logger.warning(WARNING_RUN_TRANSFORM)
                 record.run_id = None
         if hasattr(record, "transform_id") and record._meta.model_name != "run":
             record.transform = None
-            if run_context.transform is not None:
-                record.transform_id = run_context.transform.id
+            if context.run is not None:
+                record.transform_id = context.run.transform_id
             else:
                 record.transform_id = None
         # transfer other foreign key fields
@@ -528,7 +540,28 @@ def save(self, *args, **kwargs) -> Record:
     if result is not None:
         init_self_from_db(self, result)
     else:
-        super(Record, self).save(*args, **kwargs)
+        # save versioned record
+        if isinstance(self, IsVersioned) and self._is_new_version_of is not None:
+            if self._is_new_version_of.is_latest:
+                is_new_version_of = self._is_new_version_of
+            else:
+                # need one additional request
+                is_new_version_of = self.__class__.objects.get(
+                    is_latest=True, uid__startswith=self.stem_uid
+                )
+                logger.warning(
+                    f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
+                )
+            is_new_version_of.is_latest = False
+            with transaction.atomic():
+                is_new_version_of._is_new_version_of = (
+                    None  # ensure we don't start a recursion
+                )
+                is_new_version_of.save()
+                super(Record, self).save(*args, **kwargs)
+        # save unversioned record
+        else:
+            super(Record, self).save(*args, **kwargs)
     # perform transfer of many-to-many fields
     # only supported for Artifact and Collection records
     if db is not None and db != "default" and using_key is None:
@@ -553,6 +586,30 @@ def save(self, *args, **kwargs) -> Record:
     return self
 
 
+def delete(self) -> None:
+    """Delete the record."""
+    # note that the logic below does not fire if a record is moved to the trash
+    # the idea is that moving a record to the trash should move its entire version family
+    # to the trash, whereas permanently deleting should default to only deleting a single record
+    # of a version family
+    # we can consider making it easy to permanently delete entire version families as well,
+    # but that's for another time
+    if isinstance(self, IsVersioned) and self.is_latest:
+        new_latest = (
+            self.__class__.filter(is_latest=False, uid__startswith=self.stem_uid)
+            .order_by("-created_at")
+            .first()
+        )
+        if new_latest is not None:
+            new_latest.is_latest = True
+            with transaction.atomic():
+                new_latest.save()
+                super(Record, self).delete()
+            logger.warning(f"new latest version is {new_latest}")
+            return None
+    super(Record, self).delete()
+
+
 METHOD_NAMES = [
     "__init__",
     "filter",
@@ -561,6 +618,7 @@ METHOD_NAMES = [
     "search",
     "lookup",
     "save",
+    "delete",
     "from_values",
     "using",
 ]

lamindb/_run.py

@@ -42,7 +42,7 @@ def delete_run_artifacts(run: Run) -> None:
         run.save()
     if environment is not None:
         # only delete if there are no other runs attached to this environment
-        if environment.environment_of.count() == 0:
+        if environment._environment_of.count() == 0:
             environment.delete(permanent=True)
     if report is not None:
         report.delete(permanent=True)

lamindb/_transform.py

@@ -22,7 +22,6 @@ def __init__(transform: Transform, *args, **kwargs):
     is_new_version_of: Transform | None = (
         kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
     )
-    (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
@@ -55,6 +54,7 @@ def __init__(transform: Transform, *args, **kwargs):
         reference=reference,
         reference_type=reference_type,
         _has_consciously_provided_uid=has_consciously_provided_uid,
+        is_new_version_of=is_new_version_of,
     )
 
 

lamindb/core/__init__.py

@@ -35,14 +35,20 @@ Curators:
    MuDataCurator
    CurateLookup
 
-Other:
+Settings & context:
 
 .. autosummary::
    :toctree: .
 
    Settings
+   Context
+
+Data loaders:
+
+.. autosummary::
+   :toctree: .
+
    MappedCollection
-   run_context
 
 Modules:
 
@@ -84,6 +90,6 @@ from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
 from . import _data, datasets, exceptions, fields, subsettings, types
+from ._context import Context
 from ._mapped_collection import MappedCollection
-from ._run_context import run_context
 from ._settings import Settings