lamindb 1.6.1__tar.gz → 1.7a1__tar.gz

{lamindb-1.6.1 → lamindb-1.7a1}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
               docs/notes/
           )
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.2
+    rev: v0.9.10
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
@@ -51,7 +51,12 @@ repos:
     rev: v1.14.1
     hooks:
       - id: mypy
-        args: [--no-strict-optional, --ignore-missing-imports]
+        args:
+          [
+            --no-strict-optional,
+            --ignore-missing-imports,
+            --disable-error-code=annotation-unchecked,
+          ]
         additional_dependencies: ["types-requests", "types-attrs"]
         exclude: |
           (?x)(
@@ -62,5 +67,6 @@ repos:
               tests/permissions/conftest.py|
               tests/writelog/conftest.py|
               tests/writelog_sqlite/conftest.py|
-              tests/curators/test_curators_examples.py
+              tests/curators/test_curators_examples.py|
+              tests/core/conftest.py
           )

{lamindb-1.6.1 → lamindb-1.7a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.6.1
+Version: 1.7a1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -9,9 +9,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: lamin_utils==0.14.0
-Requires-Dist: lamin_cli==1.4.2
-Requires-Dist: lamindb_setup[aws]==1.6.0
+Requires-Dist: lamin_utils==0.15.0
+Requires-Dist: lamin_cli==1.5.1
+Requires-Dist: lamindb_setup[aws]==1.7.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0
@@ -23,7 +23,7 @@ Requires-Dist: anndata>=0.8.0,<=0.11.4
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: bionty>=1.5.0 ; extra == "bionty"
+Requires-Dist: bionty>=1.6.0 ; extra == "bionty"
 Requires-Dist: clinicore ; extra == "clinicore"
 Requires-Dist: tomlkit ; extra == "dev"
 Requires-Dist: line_profiler ; extra == "dev"

{lamindb-1.6.1 → lamindb-1.7a1}/docs/faq/acid.ipynb

@@ -115,7 +115,7 @@
    "id": "11",
    "metadata": {},
    "source": [
-    "This raises a `RuntimeError`:"
+    "This raises a `PermissionError`:"
    ]
   },
   {
@@ -125,7 +125,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(RuntimeError) as error:\n",
+    "with pytest.raises(PermissionError) as error:\n",
     "    artifact.save()\n",
     "print(error.exconly())"
    ]
@@ -247,7 +247,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   },
   "nbproject": {
    "id": "zz23msudiiQR",

{lamindb-1.6.1 → lamindb-1.7a1}/docs/faq/setup.ipynb

@@ -100,10 +100,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
+    "# from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
     "\n",
-    "with pytest.raises(CannotSwitchDefaultInstance):\n",
-    "    ln.setup.init(storage=\"./testsetup2\")"
+    "# with pytest.raises(CannotSwitchDefaultInstance):\n",
+    "#    ln.setup.init(storage=\"./testsetup2\")"
    ]
   },
   {
@@ -133,7 +133,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   },
   "vscode": {
    "interpreter": {

{lamindb-1.6.1 → lamindb-1.7a1}/docs/scripts/curate_soma_experiment.py

@@ -3,9 +3,8 @@ import bionty as bt
 import tiledbsoma as soma
 import tiledbsoma.io
 
-adata = ln.core.datasets.small_dataset1(otype="AnnData")
+adata = ln.core.datasets.mini_immuno.get_dataset1(otype="AnnData")
 tiledbsoma.io.from_anndata("small_dataset.tiledbsoma", adata, measurement_name="RNA")
-experiment = soma.Experiment.open("small_dataset.tiledbsoma")
 
 obs_schema = ln.Schema(
     name="soma_obs_schema",
@@ -32,11 +31,12 @@ soma_schema = ln.Schema(
     },
 ).save()
 
-curator = ln.curators.TiledbsomaExperimentCurator(experiment, soma_schema)
-curator.validate()
-artifact = curator.save_artifact(
-    key="examples/soma_experiment.tiledbsoma",
-    description="SOMA experiment with schema validation",
-)
+with soma.Experiment.open("small_dataset.tiledbsoma") as experiment:
+    curator = ln.curators.TiledbsomaExperimentCurator(experiment, soma_schema)
+    curator.validate()
+    artifact = curator.save_artifact(
+        key="examples/soma_experiment.tiledbsoma",
+        description="SOMA experiment with schema validation",
+    )
 assert artifact.schema == soma_schema
 artifact.describe()

{lamindb-1.6.1 → lamindb-1.7a1}/docs/storage/transfer-local-to-cloud.ipynb

@@ -61,7 +61,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact.save()"
+    "artifact.save(transfer=\"annotations\")"
    ]
   },
   {

{lamindb-1.6.1 → lamindb-1.7a1}/docs/transfer.ipynb

@@ -40,7 +40,7 @@
    "source": [
     "import lamindb as ln\n",
     "\n",
-    "ln.track(\"ITeOtm7bhtdq0000\")"
+    "ln.track(\"ITeOtm7bhtdq\")"
    ]
   },
   {
@@ -127,7 +127,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The artifact record and all other feature & label records have been transferred to the current database."
+    "The artifact record has been transferred to the current database without feature & label annotations, but with updated data lineage."
    ]
   },
   {
@@ -147,7 +147,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You see that the data itself remained in the original storage location, which has been added to the current instance's storage location as a read-only location."
+    "You see that the data itself remained in the original storage location, which has been added to the current instance's storage location as a read-only location (indicated by the fact that the `instance_uid` doesn't match the current instance)."
    ]
   },
   {
@@ -193,7 +193,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.view_lineage()"
@@ -209,7 +213,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.transform.description"
@@ -219,13 +227,17 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The transform key has shape `f\"transfers/{source_instance.uid}\"`:"
+    "The transform key has the form `f\"__lamindb_transfer__/{source_instance.uid}\"`:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.transform.key"
@@ -241,12 +253,78 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.run.initiated_by_run.transform"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Upon re-transferring a record, it will identify that the record already exists in the target database and simply map the record."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact = artifacts.filter(description__contains=\"Tabula Sapiens\").first()\n",
+    "artifact.save()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you also want to transfer annotations of the artifact, you can pass `transfer=\"annotations\"` to `save()`. Just note that this might populate your target database with metadata that doesn't match the conventions you want to enforce."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact = artifacts.filter(description__contains=\"Tabula Sapiens\").first()\n",
+    "artifact.save(transfer=\"annotations\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The artifact is now annotated."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.describe()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -258,11 +336,10 @@
    "outputs": [],
    "source": [
     "# test the last 3 cells here\n",
-    "# TODO restore the following test\n",
-    "# assert artifact.transform.description == \"Transfer from `laminlabs/lamindata`\"\n",
-    "# assert artifact.transform.key == \"transfers/4XIuR0tvaiXM\"\n",
-    "# assert artifact.transform.uid == \"4XIuR0tvaiXM0000\"\n",
-    "# assert artifact.run.initiated_by_run.transform.description == \"Transfer data\"\n",
+    "assert artifact.transform.description == \"Transfer from `laminlabs/lamindata`\"\n",
+    "assert artifact.transform.key == \"__lamindb_transfer__/4XIuR0tvaiXM\"\n",
+    "assert artifact.transform.uid == \"4XIuR0tvaiXM0000\"\n",
+    "assert artifact.run.initiated_by_run.transform.description == \"Transfer data\"\n",
     "\n",
     "# clean up test instance\n",
     "!lamin delete --force test-transfer"

{lamindb-1.6.1 → lamindb-1.7a1}/lamindb/__init__.py

@@ -46,7 +46,6 @@ Manage flexible records to track, e.g., samples or donors.
    :toctree: .
 
    Record
-   Sheet
 
 Manage projects.
 
@@ -109,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.6.1"
+__version__ = "1.7a1"
 
 import warnings
 
@@ -155,7 +154,6 @@ if _check_instance_setup(from_module="lamindb"):
         Space,
         Branch,
         Record,
-        Sheet,
     )
     from .models.save import save
     from . import core

{lamindb-1.6.1 → lamindb-1.7a1}/lamindb/_finish.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamin_utils._logger import LEVEL_TO_COLORS, LEVEL_TO_ICONS, RESET_COLOR
-from lamindb_setup.core.hashing import hash_file
+from lamindb_setup.core.hashing import hash_dir, hash_file
 
 from lamindb.models import Artifact, Run, Transform
 
@@ -241,6 +241,7 @@ def save_context_core(
     transform: Transform,
     filepath: Path,
     finished_at: bool = False,
+    skip_save_report: bool = False,
     ignore_non_consecutive: bool | None = None,
     from_cli: bool = False,
     is_retry: bool = False,
@@ -343,7 +344,6 @@ def save_context_core(
         transform_hash, _ = hash_file(source_code_path)  # ignore hash_type for now
         if transform.hash is not None:
             # check if the hash of the transform source code matches
-            # (for scripts, we already run the same logic in track() - we can deduplicate the call at some point)
             if transform_hash != transform.hash:
                 response = input(
                     f"You are about to overwrite existing source code (hash '{transform.hash}') for Transform('{transform.uid}')."
@@ -361,26 +361,46 @@ def save_context_core(
             transform.source_code = source_code_path.read_text()
             transform.hash = transform_hash
 
-    # track run environment
     if run is not None:
-        env_path = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
-        if env_path.exists():
+        base_path = ln_setup.settings.cache_dir / "environments" / f"run_{run.uid}"
+        paths = [base_path / "run_env_pip.txt", base_path / "r_pak_lockfile.json"]
+        existing_paths = [path for path in paths if path.exists()]
+
+        if existing_paths:
             overwrite_env = True
             if run.environment_id is not None and from_cli:
                 logger.important("run.environment is already saved, ignoring")
                 overwrite_env = False
+
             if overwrite_env:
-                env_hash, _ = hash_file(env_path)
+                # Use directory if multiple files exist, otherwise use the single file
+                artifact_path: Path = (
+                    base_path if len(existing_paths) > 1 else existing_paths[0]
+                )
+
+                # Set description based on what we're saving
+                if len(existing_paths) == 1:
+                    if existing_paths[0].name == "run_env_pip.txt":
+                        description = "requirements.txt"
+                    elif existing_paths[0].name == "r_pak_lockfile.json":
+                        description = "r_pak_lockfile.json"
+                    env_hash, _ = hash_file(artifact_path)
+                else:
+                    description = "environments"
+                    _, env_hash, _, _ = hash_dir(artifact_path)
+
                 artifact = ln.Artifact.objects.filter(hash=env_hash).one_or_none()
                 new_env_artifact = artifact is None
+
                 if new_env_artifact:
-                    artifact = ln.Artifact(  # type: ignore
-                        env_path,
-                        description="requirements.txt",
+                    artifact = ln.Artifact(
+                        artifact_path,
+                        description=description,
                         kind="__lamindb_run__",
                         run=False,
                     )
                     artifact.save(upload=True, print_progress=False)
+
                 run.environment = artifact
                 if new_env_artifact:
                     logger.debug(f"saved run.environment: {run.environment}")
@@ -394,12 +414,8 @@ def save_context_core(
         if update_finished_at:
             run.finished_at = datetime.now(timezone.utc)
 
-    # track logs
-    if run is not None and not from_cli and not is_ipynb and not is_r_notebook:
-        save_run_logs(run)
-
     # track report and set is_consecutive
-    if save_source_code_and_report:
+    if save_source_code_and_report and not skip_save_report:
         if run is not None:
             # do not save a run report if executing through nbconvert
             if report_path is not None and notebook_runner != "nbconvert":
@@ -454,7 +470,7 @@ def save_context_core(
         ln.Transform.get(transform_id_prior_to_save).delete()
 
     # finalize
-    if not from_cli and run is not None:
+    if finished_at and not from_cli and run is not None:
         run_time = run.finished_at - run.started_at
         days = run_time.days
         seconds = run_time.seconds
@@ -480,7 +496,7 @@ def save_context_core(
             logger.important(
                 f"go to: https://lamin.ai/{instance_slug}/transform/{transform.uid}"
             )
-        if not from_cli and save_source_code_and_report:
+        if finished_at and not from_cli and save_source_code_and_report:
             thing = "notebook" if (is_ipynb or is_r_notebook) else "script"
             logger.important(
                 f"to update your {thing} from the CLI, run: lamin save {filepath}"

{lamindb-1.6.1 → lamindb-1.7a1}/lamindb/base/types.py

@@ -27,7 +27,7 @@ from typing import Literal, Union
 import numpy as np
 import pandas as pd
 from django.db.models.query_utils import DeferredAttribute as FieldAttr
-from lamindb_setup.core.types import UPathStr  # noqa: F401
+from lamindb_setup.types import UPathStr  # noqa: F401
 
 # need to use Union because __future__.annotations doesn't do the job here <3.10
 # typing.TypeAlias, >3.10 on but already deprecated
@@ -37,7 +37,7 @@ StrField = Union[str, FieldAttr]  # typing.TypeAlias
 TransformType = Literal[
     "pipeline", "notebook", "upload", "script", "function", "linker"
 ]
-ArtifactKind = Literal["dataset", "model"]
+ArtifactKind = Literal["dataset", "model", "__lamindb_run__"]
 
 # below is used for Feature.dtype and Param.dtype
 Dtype = Literal[
@@ -51,10 +51,11 @@ Dtype = Literal[
     "datetime",  # datetime
     "dict",  # dictionary
     "object",  # this is a pandas input dtype, we're only using it for complicated types, not for strings
+    "path",  # path, validated as str, but specially treated in the UI
 ]
 """Data type.
 
-Data types in lamindb are a string-serialized abstraction of common data types.
+String-serialized representations of common data types.
 
 Overview
 ========
@@ -68,8 +69,9 @@ integer       `"int"`       `int64 | int32 | int16 | int8 | uint | ...`
 float         `"float"`     `float64 | float32 | float16 | float8 | ...`
 string        `"str"`       `object`
 datetime      `"datetime"`  `datetime`
-date          `"date"`      `date`
+date          `"date"`      `object` (pandera requires an ISO-format string, convert with `df["date"] = df["date"].dt.date`)
 dictionary    `"dict"`      `object`
+path          `"path"`      `str` (pandas does not have a dedicated path type, validated as `str`)
 ============  ============  =================================================
 
 Categoricals