lamindb 0.69.10__tar.gz → 0.70.0__tar.gz

{lamindb-0.69.10 → lamindb-0.70.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.10
+Version: 0.70.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.64.11
-Requires-Dist: lamindb_setup==0.69.0
+Requires-Dist: lnschema_core==0.65.0
+Requires-Dist: lamindb_setup==0.69.2
 Requires-Dist: lamin_utils==0.13.1
-Requires-Dist: lamin_cli==0.12.2
+Requires-Dist: lamin_cli==0.12.3
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0

{lamindb-0.69.10 → lamindb-0.70.0}/docs/changelog.md

@@ -1,14 +1,30 @@
 # Changelog
 
+:::{note}
+
+LaminDB implements "migration-based versioning".
+
+When upgrading your LaminDB installation to a new `minor` version in `major.minor.patch`, you also migrate your database by calling `lamin migrate deploy`.
+
+:::
+
 ```{eval-rst}
 .. role:: small
 ```
 
+## 0.70
+
+### 0.70.0 {small}`2024-04-17`
+
+- 🚸 Update data source in case transform is re-run [PR](https://github.com/laminlabs/lamindb/pull/1571) [@falexwolf](https://github.com/falexwolf)
+- 🚸 Enable to label transforms via `transform.ulabels` [PR](https://github.com/laminlabs/lnschema-core/pull/370) [@falexwolf](https://github.com/falexwolf)
+- 🚚 Deprecate `stage()` in favor of `cache()` [PR](https://github.com/laminlabs/lamindb/pull/1572) [@falexwolf](https://github.com/falexwolf)
+
 ## 0.69
 
 ### 0.69.10 {small}`2024-04-12`
 
-- ✨ Add `.obsm` and  `.layers` to `MappedCollection` [PR](https://github.com/laminlabs/lamindb/pull/1562) [@Koncopd](https://github.com/Koncopd)
+- ✨ Add `.obsm` and  `.layers` to `MappedCollection` and rename `label_keys` to `obs_keys` [PR](https://github.com/laminlabs/lamindb/pull/1562) [@Koncopd](https://github.com/Koncopd)
 - 🚸 Eliminate kwargs [PR](https://github.com/laminlabs/lamindb/pull/1561) [@sunnyosun](https://github.com/sunnyosun)
 - ✨ Introduce `Annotate.from_mudata` [PR](https://github.com/laminlabs/lamindb/pull/1554) [@sunnyosun](https://github.com/sunnyosun)
 

{lamindb-0.69.10 → lamindb-0.70.0}/docs/faq/idempotency.ipynb

@@ -31,7 +31,7 @@
     "\n",
     "If you set {attr}`~lamindb.core.Settings.upon_create_search_names` to `False`, you'll directly populate the DB.\n",
     "\n",
-    "### Files\n",
+    "### Data: artifacts & collections\n",
     "\n",
     "If you try to create a {class}`~lamindb.Artifact` object from the same content, depending on {attr}`~lamindb.core.Settings.upon_artifact_create_if_hash_exists`, \n",
     "\n",
@@ -69,7 +69,10 @@
     "import lamindb as ln\n",
     "import pytest\n",
     "\n",
-    "ln.settings.verbosity = \"hint\""
+    "ln.settings.verbosity = \"hint\"\n",
+    "ln.settings.transform.stem_uid = \"ANW20Fr4eZgM\"\n",
+    "ln.settings.transform.version = \"1\"\n",
+    "ln.track()"
    ]
   },
   {
@@ -251,7 +254,7 @@
    "id": "01bfec00",
    "metadata": {},
    "source": [
-    "### Files"
+    "### Data: artifacts and collections"
    ]
   },
   {
@@ -289,7 +292,7 @@
    "id": "a2e9114a",
    "metadata": {},
    "source": [
-    "Create a `File` object."
+    "Create an `Artifact`:"
    ]
   },
   {
@@ -323,7 +326,7 @@
    "id": "153caaea",
    "metadata": {},
    "source": [
-    "Create a `File` object from the same path:"
+    "Create an `Artifact` from the same path:"
    ]
   },
   {
@@ -333,7 +336,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact2 = ln.Artifact(filepath)"
+    "artifact2 = ln.Artifact(filepath, description=\"My fcs artifact\")"
    ]
   },
   {
@@ -352,7 +355,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert artifact.id == artifact2.id"
+    "assert artifact.id == artifact2.id\n",
+    "assert artifact.run == artifact2.run"
    ]
   },
   {
@@ -374,6 +378,32 @@
     "artifact2.save()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "ae33a6a1",
+   "metadata": {},
+   "source": [
+    "In the hidden cell below, you'll see how this interplays with data lineage."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "470d155a",
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.track(new_run=True)\n",
+    "artifact3 = ln.Artifact(filepath, description=\"My fcs artifact\")\n",
+    "assert artifact3.id == artifact2.id\n",
+    "assert artifact3.run != artifact2.run\n",
+    "assert artifact3.replicating_runs.first() == artifact2.run"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -409,7 +439,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(RuntimeError):\n",
+    "with pytest.raises(FileExistsError):\n",
     "    artifact3 = ln.Artifact(filepath, description=\"My new fcs artifact\")"
    ]
   },
@@ -522,26 +552,16 @@
    "name": "python3"
   },
   "language_info": {
-   "artifact_extension": ".py",
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
+   "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
-  },
-  "nbproject": {
-   "id": "ANW20Fr4eZgM",
-   "parent": null,
-   "pypackage": null,
-   "time_init": "2022-10-12T22:55:00.562920+00:00",
-   "user_handle": "testuser1",
-   "user_id": "DzTjkKse",
-   "user_name": "Test User1",
-   "version": "0"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

{lamindb-0.69.10 → lamindb-0.70.0}/docs/faq/track-run-inputs.ipynb

@@ -115,7 +115,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact.stage()"
+    "artifact.cache()"
    ]
   },
   {
@@ -161,7 +161,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Let us manually track an artifact by passing `is_run_input` to either `.stage()`, `.load()` or `.backed()`:"
+    "Let us manually track an artifact by passing `is_run_input` to either `.cache()`, `.load()` or `.backed()`:"
    ]
   },
   {
@@ -170,7 +170,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact.stage(is_run_input=True)"
+    "artifact.cache(is_run_input=True)"
    ]
   },
   {
@@ -217,7 +217,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "If you switch the following setting, and call to `.load()`, `.stage()` and `.backed()` will track the artifact as run input."
+    "If you switch the following setting, and call to `.load()`, `.cache()` and `.backed()` will track the artifact as run input."
    ]
   },
   {

{lamindb-0.69.10 → lamindb-0.70.0}/docs/introduction.ipynb

@@ -402,7 +402,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "# given the updated registries, the validation passes\n",
@@ -423,7 +427,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "ulabels = ln.ULabel.lookup()\n",
@@ -474,7 +482,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "cell_types.search(\"gamma delta T cell\").head(2)"
@@ -670,6 +682,9 @@
     "collection.load()\n",
     "\n",
     "# typically, it's too big, hence, iterate over its artifacts\n",
+    "collection.artifacts.all()\n",
+    "\n",
+    "# or look at a DataFrame listing the artifacts\n",
     "collection.artifacts.df()"
    ]
   },
@@ -687,12 +702,12 @@
     "```\n",
     "# to train models, batch iterate through the collection as if it was one array\n",
     "from torch.utils.data import DataLoader, WeightedRandomSampler\n",
-    "dataset = collection.mapped(label_keys=[\"perturbation\"])\n",
+    "dataset = collection.mapped(obs_keys=[\"perturbation\"])\n",
     "sampler = WeightedRandomSampler(\n",
     "    weights=dataset.get_label_weights(\"perturbation\"), num_samples=len(dataset)\n",
     ")\n",
-    "dl = DataLoader(dataset, batch_size=2, sampler=sampler)\n",
-    "for batch in dl:\n",
+    "data_loader = DataLoader(dataset, batch_size=2, sampler=sampler)\n",
+    "for batch in data_loader:\n",
     "    pass\n",
     "```"
    ]
@@ -741,7 +756,7 @@
     "\n",
     ":::\n",
     "\n",
-    "If you want to download a notebook or script, call:\n",
+    "If you want to cache a notebook or script, call:\n",
     "\n",
     "```bash\n",
     "lamin get https://lamin.ai/laminlabs/lamindata/transform/FPnfDtJz8qbE5zKv\n",

{lamindb-0.69.10 → lamindb-0.70.0}/docs/meta.ipynb

@@ -497,7 +497,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.Transform.filter(name__startswith=\"Query\").df()"
+    "ln.Transform.filter(name__startswith=\"Research\").df()"
    ]
   },
   {

{lamindb-0.69.10 → lamindb-0.70.0}/docs/storage/add-replace-stage.ipynb

@@ -5,7 +5,7 @@
    "id": "30fd690f",
    "metadata": {},
    "source": [
-    "# Add, replace, stage and delete artifacts"
+    "# Add, replace, cache and delete artifacts"
    ]
   },
   {
@@ -88,7 +88,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cache_csv_path = artifact.stage()"
+    "cache_csv_path = artifact.cache()"
    ]
   },
   {
@@ -188,7 +188,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cache_data_path = artifact.stage()"
+    "cache_data_path = artifact.cache()"
    ]
   },
   {

{lamindb-0.69.10 → lamindb-0.70.0}/docs/track.ipynb

@@ -91,11 +91,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "If you want to save a notebook including its run report & source artifact, use the CLI command:\n",
+    "If you want to save a notebook including its run report & source artifact, run:\n",
     "\n",
-    "```bash\n",
-    "\n",
-    "lamin save <notebook_artifact>\n",
+    "```python\n",
+    "ln.finish()\n",
     "```"
    ]
   },

{lamindb-0.69.10 → lamindb-0.70.0}/docs/tutorial.ipynb

@@ -407,7 +407,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To download the artifact to a local cache, call {meth}`~lamindb.Artifact.stage`:"
+    "To cache the artifact to a local cache, call {meth}`~lamindb.Artifact.cache`:"
    ]
   },
   {
@@ -416,7 +416,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact.stage()"
+    "artifact.cache()"
    ]
   },
   {
@@ -880,7 +880,7 @@
     "\n",
     "<img src=\"https://lamin-site-assets.s3.amazonaws.com/.lamindb/RGXj5wcAf7EAc6J8aBoM.png\" width=\"700px\">\n",
     "\n",
-    "If you want to download a notebook or script, call:\n",
+    "If you want to cache a notebook or script, call:\n",
     "\n",
     "```\n",
     "lamin get https://lamin.ai/laminlabs/lamindata/transform/NJvdsWWbJlZSz8\n",

{lamindb-0.69.10 → lamindb-0.70.0}/docs/tutorial2.ipynb

@@ -20,7 +20,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "In the previous tutorial ({doc}`/tutorial`), we learned about about how to leverage basic metadata for artifacts to access data (query, search, stage & load).\n",
+    "In the previous tutorial ({doc}`/tutorial`), we learned about about how to leverage basic metadata for artifacts to access data (query, search, cache & load).\n",
     "\n",
     "Here, we walk through annotating & validating data with features & labels to improve:\n",
     "\n",

{lamindb-0.69.10 → lamindb-0.70.0}/lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.10"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

{lamindb-0.69.10 → lamindb-0.70.0}/lamindb/_artifact.py

@@ -238,7 +238,7 @@ def get_stat_or_artifact(
                 "💡 you can make this error a warning:\n"
                 "    ln.settings.upon_artifact_create_if_hash_exists"
             )
-            raise RuntimeError(f"{msg}\n{hint}")
+            raise FileExistsError(f"{msg}\n{hint}")
         elif settings.upon_artifact_create_if_hash_exists == "warn_create_new":
             logger.warning(
                 "creating new Artifact object despite existing artifact with same hash:"
@@ -246,10 +246,12 @@ def get_stat_or_artifact(
             )
             return size, hash, hash_type, n_objects
         else:
-            from_trash = "(from trash)" if result[0].visibility == -1 else ""
-            logger.warning(
-                f"returning existing artifact with same hash{from_trash}: {result[0]}"
-            )
+            if result[0].visibility == -1:
+                raise FileExistsError(
+                    f"You're trying to re-create this artifact in trash: {result[0]}"
+                    "Either permanently delete it with `artifact.delete(permanent=True)` or restore it with `artifact.restore()`"
+                )
+            logger.warning(f"returning existing artifact with same hash: {result[0]}")
             return result[0]
     else:
         return size, hash, hash_type, n_objects
@@ -331,6 +333,15 @@ def get_artifact_kwargs_from_data(
         using_key=using_key,
     )
     if isinstance(stat_or_artifact, Artifact):
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if stat_or_artifact.run is not None:
+                stat_or_artifact.run.replicated_output_artifacts.add(stat_or_artifact)
+            # update the run of the artifact with the latest run
+            stat_or_artifact.run = run
+            stat_or_artifact.transform = run.transform
         return stat_or_artifact, None
     else:
         size, hash, hash_type, n_objects = stat_or_artifact
@@ -854,7 +865,7 @@ def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs)
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> Path:
+def cache(self, is_run_input: bool | None = None) -> Path:
     _track_run_input(self, is_run_input)
 
     using_key = settings._using_key
@@ -932,10 +943,10 @@ def _delete_skip_storage(artifact, *args, **kwargs) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, *args, **kwargs) -> None:
+def save(self, upload: bool | None = None, **kwargs) -> None:
     access_token = kwargs.pop("access_token", None)
 
-    self._save_skip_storage(*args, **kwargs)
+    self._save_skip_storage(**kwargs)
 
     from lamindb._save import check_and_attempt_clearing, check_and_attempt_upload
 
@@ -951,9 +962,9 @@ def save(self, *args, **kwargs) -> None:
         raise RuntimeError(exception)
 
 
-def _save_skip_storage(file, *args, **kwargs) -> None:
+def _save_skip_storage(file, **kwargs) -> None:
     save_feature_sets(file)
-    super(Artifact, file).save(*args, **kwargs)
+    super(Artifact, file).save(**kwargs)
     save_feature_set_links(file)
 
 
@@ -998,7 +1009,7 @@ METHOD_NAMES = [
     "from_df",
     "from_mudata",
     "backed",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -1024,5 +1035,6 @@ for name in METHOD_NAMES:
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
+Artifact.stage = cache
 # this seems a Django-generated function
 delattr(Artifact, "get_visibility_display")

{lamindb-0.69.10 → lamindb-0.70.0}/lamindb/_collection.py

@@ -116,6 +116,17 @@ def __init__(
         logger.warning(
             f"returning existing collection with same hash: {existing_collection}"
         )
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if existing_collection.run is not None:
+                existing_collection.run.replicated_output_collections.add(
+                    existing_collection
+                )
+            # update the run of the artifact with the latest run
+            existing_collection.run = run
+            existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         for slot, feature_set in collection.features._feature_set_by_slot.items():
             if slot in feature_sets:
@@ -227,7 +238,7 @@ def mapped(
             logger.warning(f"Ignoring artifact with suffix {artifact.suffix}")
             continue
         elif not stream:
-            path_list.append(artifact.stage())
+            path_list.append(artifact.cache())
         else:
             path_list.append(artifact.path)
     ds = MappedCollection(
@@ -248,11 +259,11 @@ def mapped(
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> list[UPath]:
+def cache(self, is_run_input: bool | None = None) -> list[UPath]:
     _track_run_input(self, is_run_input)
     path_list = []
     for artifact in self.artifacts.all():
-        path_list.append(artifact.stage())
+        path_list.append(artifact.cache())
     return path_list
 
 
@@ -355,7 +366,7 @@ def artifacts(self) -> QuerySet:
 METHOD_NAMES = [
     "__init__",
     "mapped",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -377,3 +388,4 @@ for name in METHOD_NAMES:
 # this seems a Django-generated function
 delattr(Collection, "get_visibility_display")
 Collection.artifacts = artifacts
+Collection.stage = cache