lamindb 0.76.3__tar.gz → 0.76.4__tar.gz

{lamindb-0.76.3 → lamindb-0.76.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.3
+Version: 0.76.4
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,22 +9,22 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.73.3
-Requires-Dist: lamindb_setup==0.76.7
+Requires-Dist: lnschema_core==0.73.5
+Requires-Dist: lamindb_setup==0.76.8
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.16.2
+Requires-Dist: lamin_cli==0.17.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<=0.10.8
+Requires-Dist: anndata>=0.8.0,<=0.10.9
 Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.49.0 ; extra == "bionty"
+Requires-Dist: bionty==0.49.1 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"
@@ -38,7 +38,7 @@ Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.4 ; extra == "jupyter"
-Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
+Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb

{lamindb-0.76.3 → lamindb-0.76.4}/docs/api.md

@@ -12,33 +12,19 @@ cli
 
 ```{toctree}
 :maxdepth: 1
-:caption: Core API
+:caption: Python
 :hidden:
 
 lamindb
-```
-
-```{toctree}
-:maxdepth: 1
-:caption: Plugins
-:hidden:
-
 bionty
 wetlab
 ```
 
 ```{toctree}
 :maxdepth: 1
-:caption: SDK
+:caption: R & REST
 :hidden:
 
+laminr
 rest
 ```
-
-```{toctree}
-:maxdepth: 1
-:caption: Utilities
-:hidden:
-
-readfcs
-```

lamindb-0.76.4/docs/laminr.md

@@ -0,0 +1,3 @@
+# `laminr`
+
+[github.com/laminlabs/laminr](https://github.com/laminlabs/laminr)

{lamindb-0.76.3 → lamindb-0.76.4}/docs/tutorial.ipynb

@@ -364,7 +364,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Visualize data lineage with {meth}`~lamindb.core.HasFeatures.view_lineage`:"
+    "Visualize data lineage with {meth}`~lamindb.Artifact.view_lineage`:"
    ]
   },
   {

{lamindb-0.76.3 → lamindb-0.76.4}/lamindb/__init__.py

@@ -1,7 +1,6 @@
 """A data framework for biology.
 
-Records
-=======
+Core registries.
 
 .. autosummary::
    :toctree: .
@@ -17,8 +16,7 @@ Records
    FeatureSet
    Param
 
-Key functionality
-=================
+Key functionality.
 
 .. autosummary::
    :toctree: .
@@ -29,8 +27,7 @@ Key functionality
    view
    save
 
-Modules & settings
-==================
+Modules and settings.
 
 .. autosummary::
    :toctree: .
@@ -44,7 +41,7 @@ Modules & settings
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.3"
+__version__ = "0.76.4"
 
 import os as _os
 

{lamindb-0.76.3 → lamindb-0.76.4}/lamindb/_artifact.py

@@ -366,11 +366,6 @@ def get_artifact_kwargs_from_data(
     else:
         storage = default_storage
 
-    # for now comment out this error to allow creating new versions of stores
-    # in the default folder (.lamindb)
-    #    if key is not None and key.startswith(AUTO_KEY_PREFIX):
-    #        raise ValueError(f"Key cannot start with {AUTO_KEY_PREFIX}")
-
     log_storage_hint(
         check_path_in_storage=check_path_in_storage,
         storage=storage,
@@ -542,10 +537,10 @@ def __init__(artifact: Artifact, *args, **kwargs):
         else VisibilityChoice.default.value
     )
     format = kwargs.pop("format") if "format" in kwargs else None
+    _is_internal_call = kwargs.pop("_is_internal_call", False)
     skip_check_exists = (
         kwargs.pop("skip_check_exists") if "skip_check_exists" in kwargs else False
     )
-    _uid = kwargs.pop("_uid", None)
     if "default_storage" in kwargs:
         default_storage = kwargs.pop("default_storage")
     else:
@@ -561,9 +556,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
     if "is_new_version_of" in kwargs:
         logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
         revises = kwargs.pop("is_new_version_of")
-    assert not (  # noqa: S101
-        revises is not None and _uid is not None
-    ), "Can not init with both `revises` and `_uid`"
     if not len(kwargs) == 0:
         raise ValueError(
             "Only data, key, run, description, version, revises, visibility"
@@ -579,15 +571,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError(
             f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
         )
-    if _uid is not None:
-        provisional_uid, revises = _uid, None
-    else:
-        provisional_uid, revises = create_uid(revises=revises, version=version)
     if revises is not None:
         if not isinstance(revises, Artifact):
             raise TypeError("`revises` has to be of type `Artifact`")
         if description is None:
             description = revises.description
+    if key is not None and AUTO_KEY_PREFIX in key:
+        raise ValueError(
+            f"Do not pass key that contains a managed storage path in `{AUTO_KEY_PREFIX}`"
+        )
+    # below is for internal calls that require defining the storage location
+    # ahead of constructing the Artifact
+    if isinstance(data, (str, Path)) and AUTO_KEY_PREFIX in str(data):
+        if _is_internal_call:
+            is_automanaged_path = True
+            user_provided_key = key
+            key = None
+        else:
+            raise ValueError(
+                f"Do not pass path inside the `{AUTO_KEY_PREFIX}` directory."
+            )
+    else:
+        is_automanaged_path = False
+    provisional_uid, revises = create_uid(revises=revises, version=version)
     kwargs_or_artifact, privates = get_artifact_kwargs_from_data(
         data=data,
         key=key,
@@ -615,16 +621,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    if data is not None:
+        artifact._local_filepath = privates["local_filepath"]
+        artifact._cloud_filepath = privates["cloud_filepath"]
+        artifact._memory_rep = privates["memory_rep"]
+        artifact._to_store = not privates["check_path_in_storage"]
+
+    if is_automanaged_path and _is_internal_call:
+        kwargs["_key_is_virtual"] = True
+        assert AUTO_KEY_PREFIX in kwargs["key"]  # noqa: S101
+        uid = kwargs["key"].replace(AUTO_KEY_PREFIX, "").replace(kwargs["suffix"], "")
+        kwargs["key"] = user_provided_key
+        if revises is not None:
+            assert uid.startswith(revises.stem_uid)  # noqa: S101
+        if len(uid) == 16:
+            if revises is None:
+                uid += "0000"
+            else:
+                uid, revises = create_uid(revises=revises, version=version)
+        kwargs["uid"] = uid
+
     # only set key now so that we don't do a look-up on it in case revises is passed
     if revises is not None:
         kwargs["key"] = revises.key
-    # in case we have a new version of a folder with a different hash, print a
-    # warning that the old version can't be recovered
-    if revises is not None and revises.n_objects is not None and revises.n_objects > 1:
-        logger.warning(
-            f"artifact version {version} will _update_ the state of folder {revises.path} - "
-            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `revises`"
-        )
 
     kwargs["type"] = type
     kwargs["version"] = version
@@ -643,12 +662,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
 
     add_transform_to_kwargs(kwargs, kwargs["run"])
 
-    if data is not None:
-        artifact._local_filepath = privates["local_filepath"]
-        artifact._cloud_filepath = privates["cloud_filepath"]
-        artifact._memory_rep = privates["memory_rep"]
-        artifact._to_store = not privates["check_path_in_storage"]
-
     super(Artifact, artifact).__init__(**kwargs)
 
 
@@ -943,10 +956,9 @@ def open(
                 if self.hash != hash:
                     from ._record import init_self_from_db
 
-                    logger.warning(
-                        "The hash of the tiledbsoma store has changed, creating a new version of the artifact."
-                    )
-                    new_version = Artifact(filepath, revises=self).save()
+                    new_version = Artifact(
+                        filepath, revises=self, _is_internal_call=True
+                    ).save()
                     init_self_from_db(self, new_version)
 
                     if localpath != filepath and localpath.exists():
@@ -1174,3 +1186,4 @@ Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
 Artifact.backed = backed
+Artifact.view_lineage = HasFeatures.view_lineage

{lamindb-0.76.3 → lamindb-0.76.4}/lamindb/_can_validate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable, Literal
+from typing import TYPE_CHECKING, Literal
 
 import lamindb_setup as ln_setup
 import numpy as np
@@ -79,6 +79,19 @@ def _check_organism_db(organism: Record, using_key: str | None):
                 )
 
 
+def _concat_lists(values: ListLike) -> list[str]:
+    """Concatenate a list of lists of strings into a single list."""
+    if len(values) > 0 and isinstance(values, (list, pd.Series)):
+        try:
+            if isinstance(values[0], list):
+                if isinstance(values, pd.Series):
+                    values = values.tolist()
+                values = sum([v for v in values if isinstance(v, list)], [])
+        except KeyError:
+            pass
+    return values
+
+
 def _inspect(
     cls,
     values: ListLike,
@@ -94,6 +107,7 @@ def _inspect(
 
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     queryset = _queryset(cls, using_key)
@@ -184,6 +198,7 @@ def _validate(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
 
@@ -229,7 +244,7 @@ def _validate(
 @doc_args(CanValidate.standardize.__doc__)
 def standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -295,7 +310,7 @@ def remove_synonym(self, synonym: str | ListLike):
 
 def _standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -315,6 +330,7 @@ def _standardize(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     return_field = get_name_field(
@@ -416,7 +432,7 @@ def _standardize(
 
 
 def _add_or_remove_synonyms(
-    synonym: str | Iterable,
+    synonym: str | ListLike,
     record: Record,
     action: Literal["add", "remove"],
     force: bool = False,

{lamindb-0.76.3 → lamindb-0.76.4}/lamindb/_curate.py

@@ -144,6 +144,7 @@ class DataFrameCurator(BaseCurator):
         organism: str | None = None,
         sources: dict[str, Record] | None = None,
         exclude: dict | None = None,
+        check_valid_keys: bool = True,
     ) -> None:
         from lamindb.core._settings import settings
 
@@ -163,6 +164,8 @@ class DataFrameCurator(BaseCurator):
             exclude = {}
         self._exclude = exclude
         self._non_validated = None
+        if check_valid_keys:
+            self._check_valid_keys()
         self._save_columns()
 
     @property
@@ -191,14 +194,25 @@ class DataFrameCurator(BaseCurator):
             using_key=using_key or self._using_key,
         )
 
+    def _check_valid_keys(self, extra: set = None) -> None:
+        if extra is None:
+            extra = set()
+        for name, d in {
+            "categoricals": self._fields,
+            "sources": self._sources,
+            "exclude": self._exclude,
+        }.items():
+            if not isinstance(d, dict):
+                raise TypeError(f"{name} must be a dictionary!")
+            valid_keys = set(self._df.columns) | {"columns"} | extra
+            nonval_keys = [key for key in d.keys() if key not in valid_keys]
+            if len(nonval_keys) > 0:
+                raise ValueError(
+                    f"the following keys passed to {name} are not allowed: {nonval_keys}"
+                )
+
     def _save_columns(self, validated_only: bool = True, **kwargs) -> None:
         """Save column name records."""
-        missing_columns = set(self.fields.keys()) - set(self._df.columns)
-        if missing_columns:
-            raise ValueError(
-                f"Columns {missing_columns} are not found in the data object!"
-            )
-
         # Always save features specified as the fields keys
         update_registry(
             values=list(self.fields.keys()),
@@ -426,8 +440,10 @@ class AnnDataCurator(DataFrameCurator):
             organism=organism,
             sources=sources,
             exclude=exclude,
+            check_valid_keys=False,
         )
         self._obs_fields = categoricals
+        self._check_valid_keys(extra={"var_index"})
 
     @property
     def var_index(self) -> FieldAttr:
@@ -633,6 +649,7 @@ class MuDataCurator:
                 verbosity=verbosity,
                 sources=self._sources.get(modality),
                 exclude=self._exclude.get(modality),
+                check_valid_keys=False,
                 **self._kwargs,
             )
             for modality in self._modalities

{lamindb-0.76.3 → lamindb-0.76.4}/lamindb/_finish.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import os
+import re
 import shutil
-import subprocess
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
@@ -18,12 +18,81 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 
+# this is from the get_title function in nbproject
+# should be moved into lamindb sooner or later
+def prepare_notebook(
+    nb,
+    strip_title: bool = False,
+) -> str | None:
+    """Strip title from the notebook if requested."""
+    title_found = False
+    for cell in nb.cells:
+        cell.metadata.clear()  # strip cell metadata
+        if not title_found and cell["cell_type"] == "markdown":
+            lines = cell["source"].split("\n")
+            for i, line in enumerate(lines):
+                if line.startswith("# "):
+                    line.lstrip("#").strip(" .").strip()
+                    title_found = True
+                    if strip_title:
+                        lines.pop(i)
+                        cell["source"] = "\n".join(lines)
+    return None
+
+
+def notebook_to_report(notebook_path: Path, output_path: Path) -> None:
+    import nbformat
+    import traitlets.config as config
+    from nbconvert import HTMLExporter
+
+    with open(notebook_path, encoding="utf-8") as f:
+        notebook = nbformat.read(f, as_version=4)
+    prepare_notebook(notebook, strip_title=True)
+    notebook.metadata.clear()  # strip notebook metadata
+    # if we were to export as ipynb, the following two lines would do it
+    # with open(output_path, "w", encoding="utf-8") as f:
+    #     nbformat.write(notebook, f)
+    # instead we need all this code
+    c = config.Config()
+    c.HTMLExporter.preprocessors = []
+    c.HTMLExporter.exclude_input_prompt = True
+    c.HTMLExporter.exclude_output_prompt = True
+    c.HTMLExporter.anchor_link_text = " "
+    html_exporter = HTMLExporter(config=c)
+    html, _ = html_exporter.from_notebook_node(notebook)
+    output_path.write_text(html, encoding="utf-8")
+
+
+def notebook_to_script(
+    transform: Transform, notebook_path: Path, script_path: Path
+) -> None:
+    import jupytext
+
+    notebook = jupytext.read(notebook_path)
+    py_content = jupytext.writes(notebook, fmt="py:percent")
+    # remove global metadata header
+    py_content = re.sub(r"^# ---\n.*?# ---\n\n", "", py_content, flags=re.DOTALL)
+    # replace title
+    py_content = py_content.replace(f"# # {transform.name}", "# # transform.name")
+    script_path.write_text(py_content)
+
+
+def script_to_notebook(transform: Transform, notebook_path: Path) -> None:
+    import jupytext
+
+    # get title back
+    py_content = transform.source_code.replace(
+        "# # transform.name", f"# # {transform.name}"
+    )
+    notebook = jupytext.reads(py_content, fmt="py:percent")
+    jupytext.write(notebook, notebook_path)
+
+
 def save_context_core(
     *,
     run: Run,
     transform: Transform,
     filepath: Path,
-    transform_family: QuerySet | None = None,
     finished_at: bool = False,
     from_cli: bool = False,
 ) -> str | None:
@@ -36,19 +105,17 @@ def save_context_core(
     # for scripts, things are easy
     is_consecutive = True
     is_notebook = transform.type == "notebook"
-    _source_code_artifact_path = filepath
+    source_code_path = filepath
     # for notebooks, we need more work
     if is_notebook:
         try:
-            import nbstripout
+            import jupytext
             from nbproject.dev import (
                 check_consecutiveness,
                 read_notebook,
             )
         except ImportError:
-            logger.error(
-                "install nbproject & nbstripout: pip install nbproject nbstripout"
-            )
+            logger.error("install nbproject & jupytext: pip install nbproject jupytext")
             return None
         notebook_content = read_notebook(filepath)  # type: ignore
         is_consecutive = check_consecutiveness(
@@ -62,66 +129,30 @@ def save_context_core(
                 response = "n"
             if response != "y":
                 return "aborted-non-consecutive"
-        # convert the notebook file to html
-        # log_level is set to 40 to silence the nbconvert logging
-        subprocess.run(
-            [
-                "jupyter",
-                "nbconvert",
-                "--to",
-                "html",
-                filepath.as_posix(),
-                "--Application.log_level=40",
-            ],
-            check=True,
+        # write the report
+        report_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+            ".ipynb", ".html"
         )
-        # move the temporary file into the cache dir in case it's accidentally
-        # in an existing storage location -> we want to move associated
-        # artifacts into default storage and not register them in an existing
-        # location
-        report_path_orig = filepath.with_suffix(".html")  # current location
-        report_path = ln_setup.settings.storage.cache_dir / report_path_orig.name
-        # don't use Path.rename here because of cross-device link error
-        # https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
-        shutil.move(
-            report_path_orig,  # type: ignore
-            report_path,
+        notebook_to_report(filepath, report_path)
+        # write the source code
+        source_code_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+            ".ipynb", ".py"
         )
-        # strip the output from the notebook to create the source code file
-        # first, copy the notebook file to a temporary file in the cache
-        _source_code_artifact_path = ln_setup.settings.storage.cache_dir / filepath.name
-        shutil.copy2(filepath, _source_code_artifact_path)  # copy
-        subprocess.run(
-            [
-                "nbstripout",
-                _source_code_artifact_path,
-                "--extra-keys",
-                "metadata.version metadata.kernelspec metadata.language_info metadata.pygments_lexer metadata.name metadata.file_extension",
-            ],
-            check=True,
-        )
-    # find initial versions of source codes and html reports
-    prev_report = None
-    prev_source = None
-    if transform_family is None:
-        transform_family = transform.versions
-    if len(transform_family) > 0:
-        for prev_transform in transform_family.order_by("-created_at"):
-            if (
-                prev_transform.latest_run is not None
-                and prev_transform.latest_run.report_id is not None
-            ):
-                prev_report = prev_transform.latest_run.report
-            if prev_transform._source_code_artifact_id is not None:
-                prev_source = prev_transform._source_code_artifact
+        notebook_to_script(transform, filepath, source_code_path)
     ln.settings.creation.artifact_silence_missing_run_warning = True
-
     # track source code
-    if transform._source_code_artifact_id is not None:
+    hash, _ = hash_file(source_code_path)  # ignore hash_type for now
+    if (
+        transform._source_code_artifact_id is not None
+        or transform.source_code is not None
+    ):
         # check if the hash of the transform source code matches
         # (for scripts, we already run the same logic in track() - we can deduplicate the call at some point)
-        hash, _ = hash_file(_source_code_artifact_path)  # ignore hash_type for now
-        if hash != transform._source_code_artifact.hash:
+        if transform.hash is not None:
+            condition = hash != transform.hash
+        else:
+            condition = hash != transform._source_code_artifact.hash
+        if condition:
             if os.getenv("LAMIN_TESTING") is None:
                 # in test, auto-confirm overwrite
                 response = input(
@@ -131,11 +162,8 @@ def save_context_core(
             else:
                 response = "y"
             if response == "y":
-                transform._source_code_artifact.replace(_source_code_artifact_path)
-                transform._source_code_artifact.save(upload=True)
-                logger.success(
-                    f"replaced transform._source_code_artifact: {transform._source_code_artifact}"
-                )
+                transform.source_code = source_code_path.read_text()
+                transform.hash = hash
             else:
                 logger.warning(
                     "Please re-run `ln.context.track()` to make a new version"
@@ -144,19 +172,8 @@ def save_context_core(
         else:
             logger.important("source code is already saved")
     else:
-        _source_code_artifact = ln.Artifact(
-            _source_code_artifact_path,
-            description=f"Source of transform {transform.uid}",
-            version=transform.version,
-            revises=prev_source,
-            visibility=0,  # hidden file
-            run=False,
-        )
-        _source_code_artifact.save(upload=True, print_progress=False)
-        transform._source_code_artifact = _source_code_artifact
-        logger.debug(
-            f"saved transform._source_code_artifact: {transform._source_code_artifact}"
-        )
+        transform.source_code = source_code_path.read_text()
+        transform.hash = hash
 
     # track environment
     env_path = ln_setup.settings.storage.cache_dir / f"run_env_pip_{run.uid}.txt"
@@ -211,7 +228,6 @@ def save_context_core(
             report_file = ln.Artifact(
                 report_path,
                 description=f"Report of run {run.uid}",
-                revises=prev_report,
                 visibility=0,  # hidden file
                 run=False,
             )