lamindb 1.10.1__tar.gz → 1.11.0__tar.gz

{lamindb-1.10.1 → lamindb-1.11.0}/.gitignore

@@ -66,7 +66,9 @@ run-tests
 test-django-validation/
 curate.tiledbsoma
 small_dataset.tiledbsoma
+nonregistered_storage
 registered_storage
+tests/core/notebooks/no-uid-renamed.ipynb
 
 # General
 .DS_Store
@@ -221,5 +223,5 @@ _build
 # VSCode
 .vscode
 
-# cxg
-!lamindb/examples/cellxgene/cxg_schema_versions.csv
+# CELLxGENE
+!lamindb/examples/cellxgene/cellxgene_schema_versions.csv

{lamindb-1.10.1 → lamindb-1.11.0}/.gitmodules

@@ -10,9 +10,6 @@
 [submodule "sub/wetlab"]
 	path = sub/wetlab
 	url = https://github.com/laminlabs/wetlab
-[submodule "sub/clinicore"]
-	path = sub/clinicore
-	url = https://github.com/laminlabs/clinicore
 [submodule "sub/cellxgene-lamin"]
 	path = sub/cellxgene-lamin
 	url = https://github.com/laminlabs/cellxgene-lamin.git

{lamindb-1.10.1 → lamindb-1.11.0}/.pre-commit-config.yaml

@@ -68,5 +68,6 @@ repos:
               tests/writelog/conftest.py|
               tests/writelog_sqlite/conftest.py|
               tests/curators/test_curators_examples.py|
-              tests/core/conftest.py
+              tests/core/conftest.py|
+              docs/scripts/
           )

{lamindb-1.10.1 → lamindb-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.10.1
+Version: 1.11.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -10,8 +10,14 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: lamin_utils==0.15.0
-Requires-Dist: lamin_cli==1.6.1
-Requires-Dist: lamindb_setup[aws]==1.9.1
+Requires-Dist: lamin_cli==1.7.1
+Requires-Dist: lamindb_setup[aws]==1.10.1
+Requires-Dist: bionty>=1.7a1
+Requires-Dist: wetlab>=1.5a1
+Requires-Dist: nbproject==0.11.1
+Requires-Dist: jupytext
+Requires-Dist: nbconvert>=7.2.1
+Requires-Dist: mistune!=3.1.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0
@@ -23,8 +29,6 @@ Requires-Dist: anndata>=0.8.0,<=0.12.1
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: bionty>=1.6.1rc1 ; extra == "bionty"
-Requires-Dist: clinicore>=1.2.1 ; extra == "clinicore"
 Requires-Dist: tomlkit ; extra == "dev"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
@@ -32,28 +36,19 @@ Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"
 Requires-Dist: pytest>=6.0 ; extra == "dev"
 Requires-Dist: coverage ; extra == "dev"
-Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: pytest-cov<7.0.0 ; extra == "dev"
 Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.6.0 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: pronto ; extra == "dev"
 Requires-Dist: readfcs>=2.0.1 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
-Requires-Dist: nbproject==0.11.1 ; extra == "jupyter"
-Requires-Dist: jupytext ; extra == "jupyter"
-Requires-Dist: nbconvert>=7.2.1 ; extra == "jupyter"
-Requires-Dist: mistune!=3.1.0 ; extra == "jupyter"
-Requires-Dist: wetlab>=1.3.1 ; extra == "wetlab"
 Requires-Dist: numcodecs<0.16.0 ; extra == "zarr"
 Requires-Dist: zarr>=2.16.0,<3.0.0a0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb
-Provides-Extra: bionty
-Provides-Extra: clinicore
 Provides-Extra: dev
 Provides-Extra: fcs
 Provides-Extra: gcp
-Provides-Extra: jupyter
-Provides-Extra: wetlab
 Provides-Extra: zarr
 
 [![Stars](https://img.shields.io/github/stars/laminlabs/lamindb?logo=GitHub)](https://github.com/laminlabs/lamindb)
@@ -77,7 +72,7 @@ It lets you track data transformations, validate & annotate datasets, and query
 Install the `lamindb` Python package:
 
 ```shell
-pip install 'lamindb[jupyter,bionty]'  # support notebooks & biological ontologies
+pip install lamindb
 ```
 
 Create a LaminDB instance:

{lamindb-1.10.1 → lamindb-1.11.0}/README.md

@@ -19,7 +19,7 @@ It lets you track data transformations, validate & annotate datasets, and query
 Install the `lamindb` Python package:
 
 ```shell
-pip install 'lamindb[jupyter,bionty]'  # support notebooks & biological ontologies
+pip install lamindb
 ```
 
 Create a LaminDB instance:

{lamindb-1.10.1 → lamindb-1.11.0}/docs/api.md

@@ -1,4 +1,4 @@
-# API
+# API Reference
 
 <meta http-equiv="Refresh" content="0; url=./lamindb.html" />
 
@@ -18,5 +18,4 @@ lamindb
 
 bionty
 wetlab
-clinicore
 ```

{lamindb-1.10.1 → lamindb-1.11.0}/docs/arrays.ipynb

@@ -11,7 +11,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Slice arrays"
+    "# Slice & stream arrays"
    ]
   },
   {
@@ -77,6 +77,8 @@
    "outputs": [],
    "source": [
     "import lamindb as ln\n",
+    "import numpy as np\n",
+    "import zarr\n",
     "\n",
     "ln.track(\"hsRyWJggf2Ca\")"
    ]
@@ -144,7 +146,7 @@
    },
    "outputs": [],
    "source": [
-    "adata = artifact.open()"
+    "access = artifact.open()"
    ]
   },
   {
@@ -164,7 +166,7 @@
    },
    "outputs": [],
    "source": [
-    "adata"
+    "access"
    ]
   },
   {
@@ -184,7 +186,7 @@
    },
    "outputs": [],
    "source": [
-    "adata.X"
+    "access.X"
    ]
   },
   {
@@ -204,11 +206,11 @@
    },
    "outputs": [],
    "source": [
-    "obs_idx = adata.obs.cell_type.isin([\"Dendritic cells\", \"CD14+ Monocytes\"]) & (\n",
-    "    adata.obs.percent_mito <= 0.05\n",
+    "obs_idx = access.obs.cell_type.isin([\"Dendritic cells\", \"CD14+ Monocytes\"]) & (\n",
+    "    access.obs.percent_mito <= 0.05\n",
     ")\n",
-    "adata_subset = adata[obs_idx]\n",
-    "adata_subset"
+    "access_subset = access[obs_idx]\n",
+    "access_subset"
    ]
   },
   {
@@ -228,7 +230,7 @@
    },
    "outputs": [],
    "source": [
-    "adata_subset.X"
+    "access_subset.X"
    ]
   },
   {
@@ -248,7 +250,103 @@
    },
    "outputs": [],
    "source": [
-    "adata_subset.to_memory()"
+    "adata_subset = access_subset.to_memory()\n",
+    "\n",
+    "adata_subset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Add a column to a cloud AnnData object"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is also possible to add columns to `.obs` and `.var` of cloud AnnData objects without downloading them."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a new `AnnData` `zarr` artifact."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "adata_subset.write_zarr(\"adata_subset.zarr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact = ln.Artifact(\n",
+    "    \"adata_subset.zarr\", description=\"test add column to adata\"\n",
+    ").save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "with artifact.open(mode=\"r+\") as access:\n",
+    "    access.add_column(where=\"obs\", col_name=\"ones\", col=np.ones(access.shape[0]))\n",
+    "    display(access)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The version of the artifact is updated after the modification."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact.delete(permanent=True)"
    ]
   },
   {
@@ -609,6 +707,89 @@
     "backed"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream arrays into cloud"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is also possible to write directly into the default cloud (or local) storage of the current instance and then save as an `Artifact`. This can be done using {meth}`~lamindb.Artifact.from_lazy` that returns {class}`~lamindb.models.LazyArtifact`. This object creates a real artifact on `.save()` with the provided arguments."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "lazy = ln.Artifact.from_lazy(suffix=\".zarr\", overwrite_versions=True, key=\"mydata.zarr\")\n",
+    "\n",
+    "lazy"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Stream an array into `lazy.path` in the default instance storage using `zarr`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "store = zarr.storage.FsspecStore.from_url(lazy.path.as_posix())\n",
+    "\n",
+    "group = zarr.open(store, mode=\"w\")\n",
+    "group[\"ones\"] = np.ones(3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Save and get the artifact."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact = lazy.save()\n",
+    "\n",
+    "artifact"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.delete(permanent=True)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

{lamindb-1.10.1 → lamindb-1.11.0}/docs/bio-registries.ipynb

@@ -65,7 +65,7 @@
    "outputs": [],
    "source": [
     "# [optional] inspect the available public ontology versions\n",
-    "bt.Source.df()"
+    "bt.Source.to_dataframe()"
    ]
   },
   {
@@ -110,7 +110,7 @@
    "outputs": [],
    "source": [
     "# all public cell types are now available in LaminDB\n",
-    "bt.CellType.df()"
+    "bt.CellType.to_dataframe()"
    ]
   },
   {
@@ -144,7 +144,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "bt.CellType.search(\"gamma-delta T\").df().head(2)"
+    "bt.CellType.search(\"gamma-delta T\").to_dataframe().head(2)"
    ]
   },
   {
@@ -215,7 +215,7 @@
    },
    "outputs": [],
    "source": [
-    "gdt_cell.parents.df()"
+    "gdt_cell.parents.to_dataframe()"
    ]
   },
   {
@@ -228,7 +228,7 @@
    },
    "outputs": [],
    "source": [
-    "gdt_cell.children.df()"
+    "gdt_cell.children.to_dataframe()"
    ]
   },
   {
@@ -248,7 +248,7 @@
    },
    "outputs": [],
    "source": [
-    "gdt_cell.query_parents().df()"
+    "gdt_cell.query_parents().to_dataframe()"
    ]
   },
   {
@@ -261,7 +261,7 @@
    },
    "outputs": [],
    "source": [
-    "gdt_cell.query_children().df()"
+    "gdt_cell.query_children().to_dataframe()"
    ]
   },
   {
@@ -664,7 +664,7 @@
    },
    "outputs": [],
    "source": [
-    "bt.Source.filter(currently_used=True).df()"
+    "bt.Source.filter(currently_used=True).to_dataframe()"
    ]
   },
   {
@@ -707,7 +707,7 @@
    },
    "outputs": [],
    "source": [
-    "bt.Source.filter(entity=\"bionty.Phenotype\", currently_used=True).df()"
+    "bt.Source.filter(entity=\"bionty.Phenotype\", currently_used=True).to_dataframe()"
    ]
   },
   {