lamindb 0.70.4__tar.gz → 0.71.0__tar.gz

{lamindb-0.70.4 → lamindb-0.71.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.70.4
+Version: 0.71.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.65.2
-Requires-Dist: lamindb_setup==0.70.0
+Requires-Dist: lnschema_core==0.66.0
+Requires-Dist: lamindb_setup==0.71.0
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.12.3
+Requires-Dist: lamin_cli==0.13.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,10 +23,7 @@ Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: urllib3<2 ; extra == "aws"
-Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs==2023.12.2 ; extra == "aws"
-Requires-Dist: fsspec[s3]==2023.12.2 ; extra == "aws"
+Requires-Dist: lamindb_setup[aws] ; extra == "aws"
 Requires-Dist: bionty==0.42.9 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
@@ -39,7 +36,7 @@ Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
-Requires-Dist: fsspec[gs]==2023.12.2 ; extra == "gcp"
+Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.0 ; extra == "jupyter"
 Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"

{lamindb-0.70.4 → lamindb-0.71.0}/docs/changelog.md

@@ -12,6 +12,13 @@ When upgrading your LaminDB installation to a new `minor` version in `major.mino
 .. role:: small
 ```
 
+## 0.71
+
+- ✨ Manage multiple storage locations with integrity [PR](https://github.com/laminlabs/lamindb/pull/1611) [@falexwolf](https://github.com/falexwolf)
+- 🚚 Add an `instance_uid` field to `Storage` | [374](https://github.com/laminlabs/lnschema-core/pull/374) [falexwolf](https://github.com/falexwolf)
+- 🚸 Proper progress bars for upload and download [PR](https://github.com/laminlabs/lamindb/pull/1610) [@Koncopd](https://github.com/Koncopd)
+- 🚸 Make save return self [PR](https://github.com/laminlabs/lamindb/pull/1606) [@falexwolf](https://github.com/falexwolf)
+
 ## 0.70
 
 ### 0.70.4 {small}`2024-04-24`

{lamindb-0.70.4 → lamindb-0.71.0}/docs/data.ipynb

@@ -43,42 +43,19 @@
     "In this notebook, we show how to subset an `AnnData` and generic `HDF5` and `zarr` collections accessed in the cloud."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
     "tags": [
-     "hide-output"
+     "hide-output",
+     "hide-cell"
     ]
    },
    "outputs": [],
    "source": [
-    "!lamin init --storage s3://lamindb-ci/test-data --name test-data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!lamin info"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!lamin set --auto-connect True"
+    "!lamin login testuser1\n",
+    "!lamin init --storage s3://lamindb-ci/test-array-notebook --name test-array-notebook"
    ]
   },
   {
@@ -90,15 +67,6 @@
     "import lamindb as ln"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ln.connect(\"test-data\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -337,8 +305,7 @@
    "outputs": [],
    "source": [
     "# clean up test instance\n",
-    "!lamin delete --force test-data\n",
-    "!rm -r test-data"
+    "!lamin delete --force test-array-notebook"
    ]
   }
  ],

lamindb-0.71.0/docs/faq/keep-artifacts-local.ipynb

@@ -0,0 +1,320 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Connect with `keep_artifacts_local=True`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you want to default to keeping artifacts local in a cloud instance, enable {attr}`~lamindb.setup.core.InstanceSettings.keep_artifacts_local`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "!lamin login testuser1\n",
+    "!lamin init --storage s3://lamindb-ci/keep-artifacts-local"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# the setting should be enabled on lamin.ai\n",
+    "# we're temporarily setting it here only for testing purposes\n",
+    "ln.setup.settings.instance._keep_artifacts_local = True"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can register a managed local storage location as follows:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.settings.storage_local = \"./my_storage_local\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, you have two storage locations: one in the S3 bucket, and the other locally."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.Storage.df()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you save an artifact, by default, it's stored in local storage."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "filepath = ln.core.datasets.file_fcs()\n",
+    "artifact = ln.Artifact(filepath, description=\"My fcs file\").save()\n",
+    "artifact.path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You'll see the `.fcs` file named by the `uid` in your `.lamindb/` directory under `./my_storage_local/`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.settings.storage_local.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert artifact.path.as_posix().startswith(ln.setup.settings.instance.storage_local.root.as_posix())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you'd like to upload an artifact, you pass `upload=True` to the `save()` method."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.save(upload=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You now see the artifact in the S3 bucket:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.settings.storage.root.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And it's no longer present in local storage:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.settings.instance.storage_local.root.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert artifact.path.as_posix().startswith(ln.setup.settings.instance.storage.root.as_posix())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also directly upload a file by passing `upload=True`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "filepath = ln.core.datasets.file_mini_csv()\n",
+    "artifact2 = ln.Artifact(filepath, description=\"My csv file\").save(upload=True)\n",
+    "artifact2.path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we have two files on S3:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.Artifact.df()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Delete the artifacts:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact.delete(permanent=True)\n",
+    "artifact2.delete(permanent=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Delete the instance:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.setup.delete(\"keep-artifacts-local\", force=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{lamindb-0.70.4 → lamindb-0.71.0}/docs/faq.md

@@ -12,4 +12,5 @@ faq/reference-field
 faq/key
 faq/visibility
 faq/delete
+faq/keep-artifacts-local
 ```

{lamindb-0.70.4 → lamindb-0.71.0}/docs/introduction.ipynb

@@ -158,7 +158,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Artifacts store datasets or models as files and folders."
+    "An artifact stores a dataset or model as either a file or a folder."
    ]
   },
   {
@@ -255,9 +255,8 @@
    },
    "outputs": [],
    "source": [
-    "# create a label\n",
-    "candidate_marker_study = ln.ULabel(name=\"Candidate marker study\")\n",
-    "candidate_marker_study.save()\n",
+    "# create & save a label\n",
+    "candidate_marker_study = ln.ULabel(name=\"Candidate marker study\").save()\n",
     "\n",
     "# label an artifact\n",
     "artifact.labels.add(candidate_marker_study)\n",

{lamindb-0.70.4 → lamindb-0.71.0}/docs/storage/add-replace-stage.ipynb

@@ -18,8 +18,8 @@
     "import pytest\n",
     "import lamindb as ln\n",
     "\n",
-    "INSTANCE_NAME = \"test-add-replace-stage\"\n",
-    "ln.setup.init(storage=f\"s3://lamindb-ci/{INSTANCE_NAME}\")"
+    "ln.setup.login(\"testuser1\")\n",
+    "ln.setup.init(storage=\"s3://lamindb-ci/test-add-replace-stage\")"
    ]
   },
   {
@@ -609,7 +609,6 @@
    "source": [
     "# we use the path in the next section\n",
     "path_in_storage = artifact.path\n",
-    "\n",
     "artifact.delete(permanent=True, storage=False)"
    ]
   },
@@ -764,7 +763,7 @@
    },
    "outputs": [],
    "source": [
-    "ln.setup.delete(INSTANCE_NAME, force=True)"
+    "ln.setup.delete(\"test-add-replace-stage\", force=True)"
    ]
   }
  ],

{lamindb-0.70.4 → lamindb-0.71.0}/docs/storage/upload.ipynb

@@ -19,6 +19,7 @@
     "import lamindb as ln\n",
     "import pytest\n",
     "\n",
+    "ln.setup.login(\"testuser1\")\n",
     "ln.setup.init(storage=\"s3://lamindb-ci/test-upload\")"
    ]
   },
@@ -309,7 +310,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin delete --force test-upload"
+    "ln.setup.delete(\"test-upload\", force=True)"
    ]
   }
  ],

{lamindb-0.70.4 → lamindb-0.71.0}/docs/storage/vitessce.ipynb

@@ -20,6 +20,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "!lamin login testuser1\n",
     "!lamin init --storage \"s3://lamindb-ci/test-vitessce\""
    ]
   },
@@ -134,7 +135,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!lamin delete --force test-vitessce"
+    "ln.setup.delete(\"test-vitessce\", force=True)"
    ]
   }
  ],

{lamindb-0.70.4 → lamindb-0.71.0}/docs/transfer.ipynb

@@ -11,14 +11,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Here, we'll show how to load one instance for reading data and use another to write data."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup"
+    "Here, we'll show how to transfer data from another instance into the current instance."
    ]
   },
   {
@@ -39,27 +32,12 @@
     "import lamindb as ln\n",
     "import bionty as bt\n",
     "\n",
-    "bt.settings.organism = \"human\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "bt.settings.organism = \"human\"\n",
     "ln.settings.transform.stem_uid = \"ITeOtm7bhtdq\"\n",
     "ln.settings.transform.version = \"1\"\n",
     "ln.track()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Query cellxgene"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -70,7 +48,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifacts = ln.Artifact.using(\"laminlabs/cellxgene\")\n",
@@ -87,7 +69,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact = artifacts.filter(description__icontains=\"tabula sapiens\").first()\n",
@@ -104,7 +90,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.save()"
@@ -120,7 +110,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.describe()"
@@ -130,7 +124,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The database is populated correspondingly:"
+    "The database is populated correspondingly."
    ]
   },
   {
@@ -138,7 +132,7 @@
    "execution_count": null,
    "metadata": {
     "tags": [
-     "hide-cell"
+     "hide-output"
     ]
    },
    "outputs": [],

{lamindb-0.70.4 → lamindb-0.71.0}/lamindb/__init__.py

@@ -40,7 +40,8 @@ Modules & settings:
 
 """
 
-__version__ = "0.70.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+# denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
+__version__ = "0.71.0"
 
 import os as _os