lamindb 0.70.4__tar.gz → 0.71.1__tar.gz

{lamindb-0.70.4 → lamindb-0.71.1}/.gitignore

@@ -1,4 +1,5 @@
 # LaminDB
+docsbuild/
 docs/lamin.md
 docs/guide/data-validation.ipynb
 docs/guide/knowledge.ipynb
@@ -58,6 +59,7 @@ test-search2
 test-search3
 test-search4
 test-search5
+
 # General
 .DS_Store
 

{lamindb-0.70.4 → lamindb-0.71.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.70.4
+Version: 0.71.1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.65.2
-Requires-Dist: lamindb_setup==0.70.0
+Requires-Dist: lnschema_core==0.66.3
+Requires-Dist: lamindb_setup==0.71.3
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.12.3
+Requires-Dist: lamin_cli==0.13.1
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,10 +23,7 @@ Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: urllib3<2 ; extra == "aws"
-Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs==2023.12.2 ; extra == "aws"
-Requires-Dist: fsspec[s3]==2023.12.2 ; extra == "aws"
+Requires-Dist: lamindb_setup[aws] ; extra == "aws"
 Requires-Dist: bionty==0.42.9 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
@@ -39,8 +36,8 @@ Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
-Requires-Dist: fsspec[gs]==2023.12.2 ; extra == "gcp"
-Requires-Dist: nbproject==0.10.0 ; extra == "jupyter"
+Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
+Requires-Dist: nbproject==0.10.2 ; extra == "jupyter"
 Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"

{lamindb-0.70.4 → lamindb-0.71.1}/docs/changelog.md

@@ -2,9 +2,9 @@
 
 :::{note}
 
-LaminDB implements "migration-based versioning".
+If using LaminHub, please use the latest version of lamindb.
 
-When upgrading your LaminDB installation to a new `minor` version in `major.minor.patch`, you also migrate your database by calling `lamin migrate deploy`.
+LaminDB implements "migration-based versioning". When upgrading your LaminDB installation to a new `minor` version in `major.minor.patch`, you also migrate your database by calling `lamin migrate deploy`.
 
 :::
 
@@ -12,6 +12,25 @@ When upgrading your LaminDB installation to a new `minor` version in `major.mino
 .. role:: small
 ```
 
+## 0.71
+
+### 0.71.1 {small}`2024-05-07`
+
+- 🚸  Upload source code of scripts upon `ln.finish()` and no longer upon `ln.track()` [PR](https://github.com/laminlabs/lamindb/pull/1624) [@falexwolf](https://github.com/falexwolf)
+- 🎨 Make `features.add_feature_set` public [PR](https://github.com/laminlabs/lamindb/pull/1626) [@sunnyosun](https://github.com/sunnyosun)
+- 🎨 Use the same `uid` for the same feature set in transfer [PR](https://github.com/laminlabs/lamindb/pull/1621) [@sunnyosun](https://github.com/sunnyosun)
+- 🎨 Upon upload switch to virtual key [PR](https://github.com/laminlabs/lamindb/pull/1622) [@falexwolf](https://github.com/falexwolf)
+- ⚡️ Zarr and cache improvements [PR](https://github.com/laminlabs/lamindb/pull/1620) [@Koncopd](https://github.com/Koncopd)
+- ♻️ Extend valid suffixes to composite suffixes [PR](https://github.com/laminlabs/lamindb/pull/1619) [@falexwolf](https://github.com/falexwolf)
+- 🔥 Remove little-used `artifact.view_tree()` [PR](https://github.com/laminlabs/lamindb/pull/1627) [@falexwolf](https://github.com/falexwolf)
+
+### 0.71.0 {small}`2024-05-01`
+
+- ✨ Manage multiple storage locations with integrity [PR](https://github.com/laminlabs/lamindb/pull/1611) [@falexwolf](https://github.com/falexwolf)
+- 🚚 Add an `instance_uid` field to `Storage` | [374](https://github.com/laminlabs/lnschema-core/pull/374) [falexwolf](https://github.com/falexwolf)
+- 🚸 Proper progress bars for upload and download [PR](https://github.com/laminlabs/lamindb/pull/1610) [@Koncopd](https://github.com/Koncopd)
+- 🚸 Make save return self [PR](https://github.com/laminlabs/lamindb/pull/1606) [@falexwolf](https://github.com/falexwolf)
+
 ## 0.70
 
 ### 0.70.4 {small}`2024-04-24`

{lamindb-0.70.4 → lamindb-0.71.1}/docs/data.ipynb

@@ -43,42 +43,19 @@
     "In this notebook, we show how to subset an `AnnData` and generic `HDF5` and `zarr` collections accessed in the cloud."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
     "tags": [
-     "hide-output"
+     "hide-output",
+     "hide-cell"
     ]
    },
    "outputs": [],
    "source": [
-    "!lamin init --storage s3://lamindb-ci/test-data --name test-data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!lamin info"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!lamin set --auto-connect True"
+    "!lamin login testuser1\n",
+    "!lamin init --storage s3://lamindb-ci/test-array-notebook --name test-array-notebook"
    ]
   },
   {
@@ -90,15 +67,6 @@
     "import lamindb as ln"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ln.connect(\"test-data\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -337,8 +305,7 @@
    "outputs": [],
    "source": [
     "# clean up test instance\n",
-    "!lamin delete --force test-data\n",
-    "!rm -r test-data"
+    "!lamin delete --force test-array-notebook"
    ]
   }
  ],

lamindb-0.71.1/docs/faq/keep-artifacts-local.ipynb

@@ -0,0 +1,522 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Keep artifacts local in a cloud instance"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you want to default to keeping artifacts local in a cloud instance, enable {attr}`~lamindb.setup.core.InstanceSettings.keep_artifacts_local`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "!lamin login testuser1\n",
+    "!lamin init --storage s3://lamindb-ci/keep-artifacts-local"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln\n",
+    "\n",
+    "ln.settings.transform.stem_uid = \"l9lFf83aPwRc\"\n",
+    "ln.settings.transform.version = \"1\"\n",
+    "ln.track()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# the setting should be enabled on lamin.ai\n",
+    "# we're temporarily setting it here only for testing purposes\n",
+    "ln.setup.settings.instance._keep_artifacts_local = True"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can register a managed local storage location as follows:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.settings.storage_local = \"./my_storage_local\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, you have two storage locations: one in the S3 bucket, and the other locally."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.Storage.df()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Update storage description"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can add a description to the storage by using the description parameter:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_record = ln.Storage.filter(root=ln.settings.storage_local).one()\n",
+    "storage_record.description = \"Files stored locally in site X on server Y for reason ABC\"\n",
+    "storage_record.save()\n",
+    "ln.Storage.df()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Use local storage"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you save an artifact, by default, it's stored in local storage."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "original_filepath = ln.core.datasets.file_fcs()\n",
+    "artifact = ln.Artifact(original_filepath, description=\"My fcs file\").save()\n",
+    "local_path = artifact.path\n",
+    "local_path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You'll see the `.fcs` file named by the `uid` in your `.lamindb/` directory under `./my_storage_local/`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.settings.storage_local.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert local_path.exists()\n",
+    "assert artifact.path.as_posix().startswith(ln.setup.settings.instance.storage_local.root.as_posix())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you'd like to upload an artifact, you pass `upload=True` to the `save()` method."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.save(upload=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You now see the artifact in the S3 bucket:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.settings.storage.root.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And it's no longer present in local storage:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.settings.instance.storage_local.root.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert artifact.path.exists()\n",
+    "assert not local_path.exists()\n",
+    "assert artifact.path.as_posix().startswith(ln.setup.settings.instance.storage.root.as_posix())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Direct upload"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also directly upload a file by passing `upload=True`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "filepath = ln.core.datasets.file_mini_csv()\n",
+    "artifact2 = ln.Artifact(filepath, description=\"My csv file\").save(upload=True)\n",
+    "artifact2.path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we have two files on S3:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.Artifact.df(include=\"storage__root\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert artifact2.path.exists()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Pre-existing artifacts"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Assume we already have a file in our registered local storage location:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-input"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "file_in_local_storage = ln.core.datasets.file_bam()\n",
+    "file_in_local_storage.rename(\"./my_storage_local/output.bam\")\n",
+    "ln.UPath(\"my_storage_local/\").view_tree()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If we create an artifact from it, it remains where it is during saving:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "my_existing_file = ln.Artifact(\"./my_storage_local/output.bam\", description=\"my existing file\").save()\n",
+    "ln.UPath(\"my_storage_local/\").view_tree()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The storage path of the artifact is constructed using `key` because  `key_is_virtual=False`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "my_existing_file"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "However, if we decide to upload the artifact, we'll use the `uid` for constructing the storage path and switch `key_is_virtual=True`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "my_existing_file.save(upload=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here is the remote path of the artifact:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "my_existing_file.path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And here are the contents of the storage locations:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# the path on S3\n",
+    "ln.setup.settings.storage.root.view_tree()\n",
+    "# the local path\n",
+    "ln.setup.settings.instance.storage_local.root.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Delete the test instance"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Delete the artifacts:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact.delete(permanent=True)\n",
+    "artifact2.delete(permanent=True)\n",
+    "my_existing_file.delete(permanent=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Delete the instance:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.delete(\"keep-artifacts-local\", force=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{lamindb-0.70.4 → lamindb-0.71.1}/docs/faq.md

@@ -12,4 +12,5 @@ faq/reference-field
 faq/key
 faq/visibility
 faq/delete
+faq/keep-artifacts-local
 ```