lamindb 0.76.2__py3-none-any.whl → 0.76.4__py3-none-any.whl

lamindb/_filter.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
-from lnschema_core import Artifact, Collection, Record
-from lnschema_core.types import VisibilityChoice
+from typing import TYPE_CHECKING
 
-from lamindb import settings
+from lnschema_core import Artifact, Collection
 
-from ._query_set import QuerySet
+from ._query_set import QuerySet, process_expressions
+
+if TYPE_CHECKING:
+    from lnschema_core import Record
 
 
 def filter(registry: type[Record], **expressions) -> QuerySet:
@@ -13,23 +15,7 @@ def filter(registry: type[Record], **expressions) -> QuerySet:
     _using_key = None
     if "_using_key" in expressions:
         _using_key = expressions.pop("_using_key")
-    if registry in {Artifact, Collection}:
-        # visibility is set to 0 unless expressions contains id or uid equality
-        if not (
-            "id" in expressions
-            or "uid" in expressions
-            or "uid__startswith" in expressions
-        ):
-            visibility = "visibility"
-            if not any(e.startswith(visibility) for e in expressions):
-                expressions[visibility] = (
-                    VisibilityChoice.default.value
-                )  # default visibility
-            # if visibility is None, do not apply a filter
-            # otherwise, it would mean filtering for NULL values, which doesn't make
-            # sense for a non-NULLABLE column
-            elif visibility in expressions and expressions[visibility] is None:
-                expressions.pop(visibility)
+    expressions = process_expressions(registry, expressions)
     qs = QuerySet(model=registry, using=_using_key)
     if len(expressions) > 0:
         return qs.filter(**expressions)

lamindb/_finish.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import os
+import re
 import shutil
-import subprocess
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
@@ -18,12 +18,81 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 
+# this is from the get_title function in nbproject
+# should be moved into lamindb sooner or later
+def prepare_notebook(
+    nb,
+    strip_title: bool = False,
+) -> str | None:
+    """Strip title from the notebook if requested."""
+    title_found = False
+    for cell in nb.cells:
+        cell.metadata.clear()  # strip cell metadata
+        if not title_found and cell["cell_type"] == "markdown":
+            lines = cell["source"].split("\n")
+            for i, line in enumerate(lines):
+                if line.startswith("# "):
+                    line.lstrip("#").strip(" .").strip()
+                    title_found = True
+                    if strip_title:
+                        lines.pop(i)
+                        cell["source"] = "\n".join(lines)
+    return None
+
+
+def notebook_to_report(notebook_path: Path, output_path: Path) -> None:
+    import nbformat
+    import traitlets.config as config
+    from nbconvert import HTMLExporter
+
+    with open(notebook_path, encoding="utf-8") as f:
+        notebook = nbformat.read(f, as_version=4)
+    prepare_notebook(notebook, strip_title=True)
+    notebook.metadata.clear()  # strip notebook metadata
+    # if we were to export as ipynb, the following two lines would do it
+    # with open(output_path, "w", encoding="utf-8") as f:
+    #     nbformat.write(notebook, f)
+    # instead we need all this code
+    c = config.Config()
+    c.HTMLExporter.preprocessors = []
+    c.HTMLExporter.exclude_input_prompt = True
+    c.HTMLExporter.exclude_output_prompt = True
+    c.HTMLExporter.anchor_link_text = " "
+    html_exporter = HTMLExporter(config=c)
+    html, _ = html_exporter.from_notebook_node(notebook)
+    output_path.write_text(html, encoding="utf-8")
+
+
+def notebook_to_script(
+    transform: Transform, notebook_path: Path, script_path: Path
+) -> None:
+    import jupytext
+
+    notebook = jupytext.read(notebook_path)
+    py_content = jupytext.writes(notebook, fmt="py:percent")
+    # remove global metadata header
+    py_content = re.sub(r"^# ---\n.*?# ---\n\n", "", py_content, flags=re.DOTALL)
+    # replace title
+    py_content = py_content.replace(f"# # {transform.name}", "# # transform.name")
+    script_path.write_text(py_content)
+
+
+def script_to_notebook(transform: Transform, notebook_path: Path) -> None:
+    import jupytext
+
+    # get title back
+    py_content = transform.source_code.replace(
+        "# # transform.name", f"# # {transform.name}"
+    )
+    notebook = jupytext.reads(py_content, fmt="py:percent")
+    jupytext.write(notebook, notebook_path)
+
+
 def save_context_core(
     *,
     run: Run,
     transform: Transform,
     filepath: Path,
-    transform_family: QuerySet | None = None,
     finished_at: bool = False,
     from_cli: bool = False,
 ) -> str | None:
@@ -36,23 +105,21 @@ def save_context_core(
     # for scripts, things are easy
     is_consecutive = True
     is_notebook = transform.type == "notebook"
-    _source_code_artifact_path = filepath
+    source_code_path = filepath
     # for notebooks, we need more work
     if is_notebook:
         try:
-            import nbstripout
+            import jupytext
             from nbproject.dev import (
                 check_consecutiveness,
                 read_notebook,
             )
         except ImportError:
-            logger.error(
-                "install nbproject & nbstripout: pip install nbproject nbstripout"
-            )
+            logger.error("install nbproject & jupytext: pip install nbproject jupytext")
             return None
         notebook_content = read_notebook(filepath)  # type: ignore
         is_consecutive = check_consecutiveness(
-            notebook_content, calling_statement="ln.finish()"
+            notebook_content, calling_statement=".finish()"
         )
         if not is_consecutive:
             msg = "   Do you still want to proceed with finishing? (y/n) "
@@ -62,66 +129,30 @@ def save_context_core(
                 response = "n"
             if response != "y":
                 return "aborted-non-consecutive"
-        # convert the notebook file to html
-        # log_level is set to 40 to silence the nbconvert logging
-        subprocess.run(
-            [
-                "jupyter",
-                "nbconvert",
-                "--to",
-                "html",
-                filepath.as_posix(),
-                "--Application.log_level=40",
-            ],
-            check=True,
+        # write the report
+        report_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+            ".ipynb", ".html"
         )
-        # move the temporary file into the cache dir in case it's accidentally
-        # in an existing storage location -> we want to move associated
-        # artifacts into default storage and not register them in an existing
-        # location
-        report_path_orig = filepath.with_suffix(".html")  # current location
-        report_path = ln_setup.settings.storage.cache_dir / report_path_orig.name
-        # don't use Path.rename here because of cross-device link error
-        # https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
-        shutil.move(
-            report_path_orig,  # type: ignore
-            report_path,
+        notebook_to_report(filepath, report_path)
+        # write the source code
+        source_code_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+            ".ipynb", ".py"
         )
-        # strip the output from the notebook to create the source code file
-        # first, copy the notebook file to a temporary file in the cache
-        _source_code_artifact_path = ln_setup.settings.storage.cache_dir / filepath.name
-        shutil.copy2(filepath, _source_code_artifact_path)  # copy
-        subprocess.run(
-            [
-                "nbstripout",
-                _source_code_artifact_path,
-                "--extra-keys",
-                "metadata.version metadata.kernelspec metadata.language_info metadata.pygments_lexer metadata.name metadata.file_extension",
-            ],
-            check=True,
-        )
-    # find initial versions of source codes and html reports
-    prev_report = None
-    prev_source = None
-    if transform_family is None:
-        transform_family = transform.versions
-    if len(transform_family) > 0:
-        for prev_transform in transform_family.order_by("-created_at"):
-            if (
-                prev_transform.latest_run is not None
-                and prev_transform.latest_run.report_id is not None
-            ):
-                prev_report = prev_transform.latest_run.report
-            if prev_transform._source_code_artifact_id is not None:
-                prev_source = prev_transform._source_code_artifact
+        notebook_to_script(transform, filepath, source_code_path)
     ln.settings.creation.artifact_silence_missing_run_warning = True
-
     # track source code
-    if transform._source_code_artifact_id is not None:
+    hash, _ = hash_file(source_code_path)  # ignore hash_type for now
+    if (
+        transform._source_code_artifact_id is not None
+        or transform.source_code is not None
+    ):
         # check if the hash of the transform source code matches
         # (for scripts, we already run the same logic in track() - we can deduplicate the call at some point)
-        hash, _ = hash_file(_source_code_artifact_path)  # ignore hash_type for now
-        if hash != transform._source_code_artifact.hash:
+        if transform.hash is not None:
+            condition = hash != transform.hash
+        else:
+            condition = hash != transform._source_code_artifact.hash
+        if condition:
             if os.getenv("LAMIN_TESTING") is None:
                 # in test, auto-confirm overwrite
                 response = input(
@@ -131,11 +162,8 @@ def save_context_core(
             else:
                 response = "y"
             if response == "y":
-                transform._source_code_artifact.replace(_source_code_artifact_path)
-                transform._source_code_artifact.save(upload=True)
-                logger.success(
-                    f"replaced transform._source_code_artifact: {transform._source_code_artifact}"
-                )
+                transform.source_code = source_code_path.read_text()
+                transform.hash = hash
             else:
                 logger.warning(
                     "Please re-run `ln.context.track()` to make a new version"
@@ -144,19 +172,8 @@ def save_context_core(
         else:
             logger.important("source code is already saved")
     else:
-        _source_code_artifact = ln.Artifact(
-            _source_code_artifact_path,
-            description=f"Source of transform {transform.uid}",
-            version=transform.version,
-            revises=prev_source,
-            visibility=0,  # hidden file
-            run=False,
-        )
-        _source_code_artifact.save(upload=True, print_progress=False)
-        transform._source_code_artifact = _source_code_artifact
-        logger.debug(
-            f"saved transform._source_code_artifact: {transform._source_code_artifact}"
-        )
+        transform.source_code = source_code_path.read_text()
+        transform.hash = hash
 
     # track environment
     env_path = ln_setup.settings.storage.cache_dir / f"run_env_pip_{run.uid}.txt"
@@ -211,7 +228,6 @@ def save_context_core(
             report_file = ln.Artifact(
                 report_path,
                 description=f"Report of run {run.uid}",
-                revises=prev_report,
                 visibility=0,  # hidden file
                 run=False,
             )

lamindb/_query_set.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Iterable, NamedTuple
 import pandas as pd
 from django.db import models
 from django.db.models import F
+from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core.models import (
     Artifact,
@@ -13,8 +14,10 @@ from lnschema_core.models import (
     Collection,
     IsVersioned,
     Record,
+    Registry,
     Run,
     Transform,
+    VisibilityChoice,
 )
 
 from lamindb.core.exceptions import DoesNotExist
@@ -64,6 +67,27 @@ def one_helper(self):
         return self[0]
 
 
+def process_expressions(registry: Registry, expressions: dict) -> dict:
+    if registry in {Artifact, Collection}:
+        # visibility is set to 0 unless expressions contains id or uid equality
+        if not (
+            "id" in expressions
+            or "uid" in expressions
+            or "uid__startswith" in expressions
+        ):
+            visibility = "visibility"
+            if not any(e.startswith(visibility) for e in expressions):
+                expressions[visibility] = (
+                    VisibilityChoice.default.value
+                )  # default visibility
+            # if visibility is None, do not apply a filter
+            # otherwise, it would mean filtering for NULL values, which doesn't make
+            # sense for a non-NULLABLE column
+            elif visibility in expressions and expressions[visibility] is None:
+                expressions.pop(visibility)
+    return expressions
+
+
 def get(
     registry_or_queryset: type[Record] | QuerySet,
     idlike: int | str | None = None,
@@ -88,7 +112,7 @@ def get(
             return qs.one()
     else:
         assert idlike is None  # noqa: S101
-        # below behaves exactly like `.one()`
+        expressions = process_expressions(registry, expressions)
         return registry.objects.get(**expressions)
 
 
@@ -108,7 +132,7 @@ class RecordsList(UserList):
         return one_helper(self)
 
 
-class QuerySet(models.QuerySet, CanValidate):
+class QuerySet(models.QuerySet):
     """Sets of records returned by queries.
 
     See Also:
@@ -221,6 +245,7 @@ class QuerySet(models.QuerySet, CanValidate):
         # both Transform & Run might reference artifacts
         if self.model in {Artifact, Collection, Transform, Run}:
             for record in self:
+                logger.important(f"deleting {record}")
                 record.delete(*args, **kwargs)
         else:
             self._delete_base_class(*args, **kwargs)
@@ -276,42 +301,50 @@ class QuerySet(models.QuerySet, CanValidate):
         else:
             raise ValueError("Record isn't subclass of `lamindb.core.IsVersioned`")
 
-    @doc_args(Record.search.__doc__)
-    def search(self, string: str, **kwargs):
-        """{}"""  # noqa: D415
-        from ._record import _search
 
-        return _search(cls=self, string=string, **kwargs)
+# -------------------------------------------------------------------------------------
+# CanValidate
+# -------------------------------------------------------------------------------------
 
-    @doc_args(Record.lookup.__doc__)
-    def lookup(self, field: StrField | None = None, **kwargs) -> NamedTuple:
-        """{}"""  # noqa: D415
-        from ._record import _lookup
 
-        return _lookup(cls=self, field=field, **kwargs)
+@doc_args(Record.search.__doc__)
+def search(self, string: str, **kwargs):
+    """{}"""  # noqa: D415
+    from ._record import _search
 
-    @doc_args(CanValidate.validate.__doc__)
-    def validate(self, values: ListLike, field: str | StrField | None = None, **kwargs):
-        """{}"""  # noqa: D415
-        from ._can_validate import _validate
+    return _search(cls=self, string=string, **kwargs)
 
-        return _validate(cls=self, values=values, field=field, **kwargs)
 
-    @doc_args(CanValidate.inspect.__doc__)
-    def inspect(self, values: ListLike, field: str | StrField | None = None, **kwargs):
-        """{}"""  # noqa: D415
-        from ._can_validate import _inspect
+@doc_args(Record.lookup.__doc__)
+def lookup(self, field: StrField | None = None, **kwargs) -> NamedTuple:
+    """{}"""  # noqa: D415
+    from ._record import _lookup
 
-        return _inspect(cls=self, values=values, field=field, **kwargs)
+    return _lookup(cls=self, field=field, **kwargs)
 
-    @doc_args(CanValidate.standardize.__doc__)
-    def standardize(
-        self, values: Iterable, field: str | StrField | None = None, **kwargs
-    ):
-        """{}"""  # noqa: D415
-        from ._can_validate import _standardize
 
-        return _standardize(cls=self, values=values, field=field, **kwargs)
+@doc_args(CanValidate.validate.__doc__)
+def validate(self, values: ListLike, field: str | StrField | None = None, **kwargs):
+    """{}"""  # noqa: D415
+    from ._can_validate import _validate
+
+    return _validate(cls=self, values=values, field=field, **kwargs)
+
+
+@doc_args(CanValidate.inspect.__doc__)
+def inspect(self, values: ListLike, field: str | StrField | None = None, **kwargs):
+    """{}"""  # noqa: D415
+    from ._can_validate import _inspect
+
+    return _inspect(cls=self, values=values, field=field, **kwargs)
+
+
+@doc_args(CanValidate.standardize.__doc__)
+def standardize(self, values: Iterable, field: str | StrField | None = None, **kwargs):
+    """{}"""  # noqa: D415
+    from ._can_validate import _standardize
+
+    return _standardize(cls=self, values=values, field=field, **kwargs)
 
 
 models.QuerySet.df = QuerySet.df
@@ -320,10 +353,10 @@ models.QuerySet.first = QuerySet.first
 models.QuerySet.one = QuerySet.one
 models.QuerySet.one_or_none = QuerySet.one_or_none
 models.QuerySet.latest_version = QuerySet.latest_version
-models.QuerySet.search = QuerySet.search
-models.QuerySet.lookup = QuerySet.lookup
-models.QuerySet.validate = QuerySet.validate
-models.QuerySet.inspect = QuerySet.inspect
-models.QuerySet.standardize = QuerySet.standardize
+models.QuerySet.search = search
+models.QuerySet.lookup = lookup
+models.QuerySet.validate = validate
+models.QuerySet.inspect = inspect
+models.QuerySet.standardize = standardize
 models.QuerySet._delete_base_class = models.QuerySet.delete
 models.QuerySet.delete = QuerySet.delete

lamindb/_record.py

@@ -57,7 +57,7 @@ def suggest_records_with_similar_names(record: Record, kwargs) -> bool:
     if kwargs.get("name") is None:
         return False
     queryset = _search(
-        record.__class__, kwargs["name"], field="name", truncate_words=True, limit=20
+        record.__class__, kwargs["name"], field="name", truncate_words=True, limit=3
     )
     if not queryset.exists():  # empty queryset
         return False
@@ -586,7 +586,8 @@ def delete(self) -> None:
     # but that's for another time
     if isinstance(self, IsVersioned) and self.is_latest:
         new_latest = (
-            self.__class__.filter(is_latest=False, uid__startswith=self.stem_uid)
+            self.__class__.objects.using(self._state.db)
+            .filter(is_latest=False, uid__startswith=self.stem_uid)
             .order_by("-created_at")
             .first()
         )

lamindb/_transform.py

@@ -37,8 +37,7 @@ def __init__(transform: Transform, *args, **kwargs):
             "Only name, key, version, type, revises, reference, "
             f"reference_type can be passed, but you passed: {kwargs}"
         )
-    # Transform allows passing a uid, all others don't
-    if uid is None and key is not None:
+    if revises is None and key is not None:
         revises = Transform.filter(key=key).order_by("-created_at").first()
     if revises is not None and key is not None and revises.key != key:
         note = message_update_key_in_version_family(

lamindb/core/__init__.py

@@ -10,8 +10,6 @@ Registries:
    QuerySet
    QueryManager
    RecordsList
-   HasFeatures
-   HasParams
    FeatureManager
    ParamManager
    LabelManager
@@ -30,6 +28,7 @@ Curators:
 .. autosummary::
    :toctree: .
 
+   BaseCurator
    DataFrameCurator
    AnnDataCurator
    MuDataCurator
@@ -80,6 +79,7 @@ from lnschema_core.models import (
 
 from lamindb._curate import (
     AnnDataCurator,
+    BaseCurator,
     CurateLookup,
     DataFrameCurator,
     MuDataCurator,

lamindb/core/_context.py

@@ -18,7 +18,7 @@ from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
 from .exceptions import (
     MissingContext,
-    NotebookNotSaved,
+    NotebookFileNotSavedToDisk,
     NotebookNotSavedError,
     NoTitleError,
     TrackNotCalled,
@@ -414,6 +414,10 @@ class Context:
         if transform is None:
             if uid is None:
                 uid = f"{stem_uid}{get_uid_ext(version)}"
+            # note that here we're not passing revises because we're not querying it
+            # hence, we need to do a revision family lookup based on key
+            # hence, we need key to be not None
+            assert key is not None  # noqa: S101
             transform = Transform(
                 uid=uid,
                 version=version,
@@ -422,8 +426,7 @@ class Context:
                 reference=transform_ref,
                 reference_type=transform_ref_type,
                 type=transform_type,
-            )
-            transform.save()
+            ).save()
             self._logging_message += f"created Transform('{transform.uid}')"
         else:
             uid = transform.uid
@@ -449,29 +452,36 @@ class Context:
                     "updated transform name, "  # white space on purpose
                 )
             # check whether transform source code was already saved
-            if transform._source_code_artifact_id is not None:
-                response = None
+            if (
+                transform._source_code_artifact_id is not None
+                or transform.source_code is not None
+            ):
+                bump_revision = False
                 if is_run_from_ipython:
-                    response = "y"  # auto-bump version
+                    bump_revision = True
                 else:
                     hash, _ = hash_file(self._path)  # ignore hash_type for now
-                    if hash != transform._source_code_artifact.hash:
-                        response = "y"  # auto-bump version
+                    if transform.hash is not None:
+                        condition = hash != transform.hash
+                    else:
+                        condition = hash != transform._source_code_artifact.hash
+                    if condition:
+                        bump_revision = True
                     else:
                         self._logging_message += f"loaded Transform('{transform.uid}')"
-                if response is not None:
+                if bump_revision:
                     change_type = (
                         "Re-running saved notebook"
                         if is_run_from_ipython
                         else "Source code changed"
                     )
                     suid, vuid = (
-                        uid[: Transform._len_stem_uid],
-                        uid[Transform._len_stem_uid :],
+                        uid[:-4],
+                        uid[-4:],
                     )
                     new_vuid = increment_base62(vuid)
                     raise UpdateContext(
-                        f"{change_type}, bump version by setting:\n\n"
+                        f"{change_type}, bump revision by setting:\n\n"
                         f'ln.context.uid = "{suid}{new_vuid}"'
                     )
             else:
@@ -504,8 +514,8 @@ class Context:
                 get_seconds_since_modified(context._path) > 3
                 and os.getenv("LAMIN_TESTING") is None
             ):
-                raise NotebookNotSaved(
-                    "Please save the notebook in your editor right before running `ln.finish()`"
+                raise NotebookFileNotSavedToDisk(
+                    "Please save the notebook manually in your editor right before running `ln.finish()`"
                 )
         save_context_core(
             run=context.run,

lamindb/core/_label_manager.py

@@ -118,13 +118,11 @@ def validate_labels(labels: QuerySet | list | dict):
 
 
 class LabelManager:
-    """Label manager (:attr:`~lamindb.core.HasFeatures.labels`).
+    """Label manager.
 
     This allows to manage untyped labels :class:`~lamindb.ULabel` and arbitrary
     typed labels (e.g., :class:`~bionty.CellLine`) and associate labels
     with features.
-
-    See :class:`~lamindb.core.HasFeatures` for more information.
     """
 
     def __init__(self, host: Artifact | Collection):

lamindb/core/_mapped_collection.py

@@ -17,6 +17,7 @@ from .storage._anndata_accessor import (
     GroupTypes,
     StorageType,
     _safer_read_index,
+    get_spec,
     registry,
 )
 
@@ -153,13 +154,30 @@ class MappedCollection:
         self._make_connections(path_list, parallel)
 
         self.n_obs_list = []
-        for storage in self.storages:
+        for i, storage in enumerate(self.storages):
             with _Connect(storage) as store:
                 X = store["X"]
+                store_path = self.path_list[i]
+                self._check_csc_raise_error(X, "X", store_path)
                 if isinstance(X, ArrayTypes):  # type: ignore
                     self.n_obs_list.append(X.shape[0])
                 else:
                     self.n_obs_list.append(X.attrs["shape"][0])
+                for layer_key in self.layers_keys:
+                    if layer_key == "X":
+                        continue
+                    self._check_csc_raise_error(
+                        store["layers"][layer_key],
+                        f"layers/{layer_key}",
+                        store_path,
+                    )
+                if self.obsm_keys is not None:
+                    for obsm_key in self.obsm_keys:
+                        self._check_csc_raise_error(
+                            store["obsm"][obsm_key],
+                            f"obsm/{obsm_key}",
+                            store_path,
+                        )
         self.n_obs = sum(self.n_obs_list)
 
         self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
@@ -281,6 +299,18 @@ class MappedCollection:
         vars = pd.Index(vars)
         return [i for i, vrs in enumerate(self.var_list) if not vrs.equals(vars)]
 
+    def _check_csc_raise_error(
+        self, elem: GroupType | ArrayType, key: str, path: UPathStr
+    ):
+        if isinstance(elem, ArrayTypes):  # type: ignore
+            return
+        if get_spec(elem).encoding_type == "csc_matrix":
+            if not self.parallel:
+                self.close()
+            raise ValueError(
+                f"{key} in {path} is a csc matrix, `MappedCollection` doesn't support this format yet."
+            )
+
     def __len__(self):
         return self.n_obs